refspy.matcher

  1"""Match biblical references in texts, returning strings and Reference objects."""
  2
  3import math
  4import re
  5from re import Match
  6from collections.abc import Generator
  7
  8from refspy.models.book import Book
  9from refspy.models.language import Language
 10from refspy.models.range import Range, range, verse_range
 11from refspy.models.reference import (
 12    Reference,
 13    book_reference,
 14    reference,
 15)
 16from refspy.models.syntax import Syntax
 17from refspy.models.verse import verse
 18
 19from refspy.types.number import Number
 20
 21from refspy.utils import (
 22    add_space_after_book_number,
 23    get_unnumbered_book_aliases,
 24    parse_number,
 25    normalize_spacing,
 26    trim_trailing_period,
 27)
 28
 29
 30class Matcher:
 31    """
 32    Match:
 33    - Opening and closing brackets
 34      - So we don't carry book context past the end of parentheses
 35    - [PROPOSED] Ends of sentence and end of paragraph.
 36      - So we don't carry book context for an undue distance within the text.
 37    - References
 38      - Book names (incl. substitutions like 'First' for '1':
 39        - with no reference: sets context; only returned as a reference if
 40            requested with yield_books=True
 41        - when book.chapter > 1:
 42          - 1:2-3:4
 43          - 1:1,2-3 etc
 44        - when book.chapter == 1:
 45          - 1,2-4  (can be verses or chapters, based on book; must not end with
 46            a number that starts a book name)
 47      - Stand-alone number references: always have colons or verse markers.
 48        They rely on context being provided by a previous reference or book
 49        name.
 50        - v.1, vv.1-2
 51        - 1:2-3:4
 52        - 1:1,2-3 etc (must not end with a number that starts a book name)
 53    Yield:
 54      - a (match_str, reference) tuple for each match.
 55    """
 56
 57    def __init__(
 58        self,
 59        books: dict[tuple[Number, Number], Book],
 60        book_aliases: dict[str, tuple[Number, Number]],
 61        language: Language,
 62        syntax: Syntax | None = None,
 63    ):
 64        self.books = books
 65        self.language = language
 66        self.syntax = syntax or language.syntax
 67        self.book_alias_keys = book_aliases.keys()
 68        self.book_aliases = self.expand_book_aliases(book_aliases)
 69
 70        # Regexes for matching are generated on intiialisation,
 71        # using the language object for punctuation.
 72
 73        self.SPACE = r"\s+"
 74        self.OPTIONAL_SPACE = r"\s*"
 75        self.END = r"\b"
 76
 77        self.COMMA = (
 78            "[" + re.escape(self.syntax.match_commas) + "]" + self.OPTIONAL_SPACE
 79        )
 80        self.COLON = (
 81            "[" + re.escape(self.syntax.match_colons) + "]" + self.OPTIONAL_SPACE
 82        )
 83        self.DASH = "[" + re.escape(self.syntax.match_dashes) + "]"
 84        self.NUMBER = r"\d+[a-d]?"
 85
 86        self.RANGE = f"{self.NUMBER}{self.DASH}{self.NUMBER}"
 87        self.LIST = f"(?:{self.RANGE}|{self.NUMBER})(?:{self.COMMA}(?:{self.RANGE}|{self.NUMBER}))*"
 88
 89        self.RANGE_OR_NUMBER_COMPILED = re.compile(f"({self.RANGE}|{self.NUMBER})")
 90
 91        self.NUMBER_CAPTURE = re.compile(f"({self.NUMBER})")
 92        self.RANGE_CAPTURE = re.compile(f"({self.NUMBER}){self.DASH}({self.NUMBER})")
 93        self.CHAPTER_RANGE_CAPTURE = re.compile(
 94            f"{self.END}({self.NUMBER}){self.COLON}({self.NUMBER}){self.DASH}"
 95            + f"({self.NUMBER}){self.COLON}({self.NUMBER}){self.END}"
 96        )
 97        self.CHAPTER_VERSES_CAPTURE = re.compile(
 98            f"{self.END}({self.NUMBER}){self.COLON}({self.LIST}){self.END}"
 99        )
100
101        self.brackets_regexp = re.compile(self.build_brackets_regexp())
102        self.reference_regexp = re.compile(self.build_reference_regexp())
103
104    def build_reference_regexp(self) -> str:
105        """
106        Reference matches are quadruples. For the string ...
107
108        "Big Book 1:2-5 (cf. Small Book 34) is more interesting than vv.2:3-6."
109
110        ... We want to know which references are attached to books and which are just numeric:
111
112            ```
113            Matching string
114            :                 Book Name
115            :                  :            Book Reference
116            :                  :             :        Numeric Reference
117            :                  :             :         :
118            ('Big Book 1:2-5', 'Big Book',   '1:2-5',  None)
119            ('Small Book 34',  'Small Book', '34',     None)
120            ( None,             None,         None,   '2:3-6')
121            ```
122
123        Additionally, we want to minimize bad matches like John Smith, or John
124        B. Smith; a negative lookahead for a following capital eliminates a lot
125        of false positives; may be necessary to treat known names differently
126        in each language.
127        """
128        NAME_PATTERN = self.build_book_name_regexp()
129        VERSE_MARKER = self.build_verse_marker_regexp()
130        NUMBER_LIST = self.build_number_list_regexp()
131        REGEXP = "".join(
132            [
133                "(",
134                f"{self.END}({NAME_PATTERN})(?!{self.SPACE}[A-Z])",  # John, but not John B.
135                "(",
136                "".join(
137                    [
138                        f"{self.OPTIONAL_SPACE}{self.NUMBER}{self.COLON}{self.NUMBER}{self.DASH}{self.NUMBER}{self.COLON}{self.NUMBER}",  # Rom 1:2-3:4
139                        "|",
140                        f"{self.OPTIONAL_SPACE}{self.NUMBER}{self.COLON}{NUMBER_LIST}",  # Rom 3:4,6-9
141                        "|",
142                        f"{self.OPTIONAL_SPACE}{NUMBER_LIST}",  # Phlm 3-4 (verse), Rom 3-4 (chapter)
143                    ]
144                ),
145                ")?",
146                ")|(",
147                "".join(
148                    [
149                        f"{self.NUMBER}{self.COLON}{self.NUMBER}{self.DASH}{self.NUMBER}{self.COLON}{self.NUMBER}",  # 1:2-3:4
150                        "|",
151                        f"{self.NUMBER}{self.COLON}{NUMBER_LIST}",  # 3:4,6-9
152                        "|",
153                        f"(?:{VERSE_MARKER}){NUMBER_LIST}",  # vv.3-4
154                    ]
155                ),
156                ")",
157            ]
158        )
159        return REGEXP
160
161    def expand_book_aliases(
162        self, aliases: dict[str, tuple[int, int]]
163    ) -> dict[str, tuple[int, int]]:
164        """
165        SBL style requires e.g. "1 Corinthians" be recognized as "First
166        Corinthians" at the start of a sentence. Add these as keys to
167        the aliases.
168
169        Note:
170            Number prefixes are set in the Language class.
171        """
172        out = {}
173        for alias, library_book in aliases.items():
174            out[alias] = library_book
175            for number, prefixes in self.language.number_prefixes.items():
176                if alias.startswith(number + " "):
177                    for prefix in prefixes:
178                        out[prefix + alias[1:]] = library_book
179        return out
180
181    def build_book_name_regexp(self):
182        """
183        Sort bookname DESC to match the longest first.
184        Replace spaces with multi-space matchers in book names.
185        Group book names by prefixes.
186        Match substitute prefixes for each prefix number:
187
188        Example:
189            ```
190            (   (1|First|etc)\\s*(Corinthians|etc)
191              | (2|Second|etc)\\s*(Corinthians|etc)
192              |   ...
193              | (Galatians|Romans|etc)
194            )
195            ```
196        """
197        regexp_parts = []
198        for number in self.numerical_book_prefixes(reverse=True):
199            aliases = [
200                alias[len(number + " ") :]
201                for alias in self.book_alias_keys
202                if alias.startswith(number + " ")
203            ]
204            regexp_parts.append(
205                r"(?:"
206                + "|".join(
207                    [number]
208                    + [
209                        escape_book_name(_)
210                        for _ in self.language.number_prefixes[number]
211                    ]
212                )
213                + r")"
214                + self.OPTIONAL_SPACE
215                + "(?:"
216                + "|".join([escape_book_name(_) for _ in long_names_first(aliases)])
217                + ")"
218                # + r"(?![^A-Za-z])"  # <-- Need a non-alpha lookahead?
219                + r"\b"
220                + r"\.?"
221            )
222        leading_digits = re.compile(r"^\d+\s")
223        aliases = [
224            alias for alias in self.book_alias_keys if not leading_digits.match(alias)
225        ]
226        regexp_parts.append(
227            r"(?:"
228            + "|".join([escape_book_name(_) for _ in long_names_first(aliases)])
229            + ")"
230            # + r"(?![^A-Za-z])"  # <-- Need a non-alpha lookahead?
231            + r"\b"
232            + r"\.?"
233        )
234        return r"|".join(regexp_parts)
235
236    def build_verse_marker_regexp(self):
237        return "|".join(
238            [
239                re.escape(key) + self.OPTIONAL_SPACE
240                for key in self.language.verse_markers
241            ]
242        )
243
244    def build_number_list_regexp(self):
245        """
246        Match a comma-(and-space?)-separated list of numbers and ranges.
247
248        A VERSE NUMBER is a number that can't be the start of a book name or chapter reference.
249
250        We use negative lookaheads to exclude book names (1 Cor) or chapter refs (6:5):
251
252        For en_US, this will look like:
253
254            ```
255            (
256                1(?![:.]|\\s*(?:Corinthians|Cor|Thess|etc))
257              | 2(?![:.]|\\s*(?:Corinthians|Cor|Thess|etc))
258              | 3(?![:.]|\\s*(?:Jn))
259              | [4-999](?![:.])
260            )
261            ```
262        """
263        regexp_parts = []
264        # numbers which can be book numbers, when not followed by ':' or book names
265        for number in self.numerical_book_prefixes():
266            aliases = [
267                alias[len(number + " ") :]
268                for alias in self.book_aliases.keys()
269                if alias.startswith(number + " ")
270            ]
271            regexp_parts.append(
272                number
273                + r"(?!"  # <-- negative look-ahead
274                + self.COLON
275                + r"|"
276                + r"\s*"
277                + "(?:"  # <-- non-capturing group
278                + "|".join([re.escape(key) for key in sorted(set(aliases))])
279                + "))"
280            )
281
282        # TODO: REMOVE (after merge)
283
284        # numbers which cannot be book numbers, when not followed by ':'
285        # if it's 4..999, then in regexp that's 4-9 or a 2 or 3 digit number
286        regexp_parts.append(
287            f"(?:[{len(regexp_parts) + 1}-9]|" + r"\d{2,3}" + f")(?!{self.COLON})"
288        )
289        VERSE_NUMBER = r"|".join(regexp_parts)
290        REGEXP = f"(?:{self.RANGE}|{self.NUMBER})(?:{self.COMMA}(?:{self.RANGE}|(?:{VERSE_NUMBER}){self.END}))*"
291        return REGEXP
292        # regexp_parts.append(f"[{len(regexp_parts) + 1}-999]")
293        # SAFE_NUMBER = r"|".join(regexp_parts)
294        # REGEXP = f"(?:{self.RANGE}|{self.NUMBER})(?:{self.COMMA}{self.SPACE}(?:{self.RANGE}|(?:{SAFE_NUMBER}){self.END}))*"
295
296    def numerical_book_prefixes(self, reverse=True) -> list[str]:
297        prefixes = set()
298        for key in self.book_aliases.keys():
299            part_1 = key.split(" ")[0]
300            if part_1.isdigit():
301                prefixes.add(part_1)
302        return sorted(prefixes, reverse=reverse)
303
304    def build_brackets_regexp(self) -> str:
305        return r"[\(\)]"
306
307    def generate_references(
308        self,
309        text: str,
310        yield_books: bool = False,
311        yield_nones: bool = False,
312        use_context: bool = True,
313    ) -> Generator[tuple[str, Reference | None], None, None]:
314        """
315        Match references and parentheses separately, then take the next lowest
316        item (by starting match position) from the regexp match generators, and
317        process. Yield references for book names and chapter/verse numbers.
318
319        This is the base function used by `__.find_references()` and
320        `__.first_reference()`. It keeps all the logic in one place.
321
322        Args:
323            text: In which to find references
324            yield_books: Whether to match book names alone
325            yield_nones: Whether to match malformed references
326            use_context: Whether to use context for number-only references.
327        """
328        brackets_matches = self.brackets_regexp.finditer(text)
329        reference_matches = self.reference_regexp.finditer(text)
330
331        bracket_stack = []
332
333        brackets_match = next(brackets_matches, None)
334        reference_match = next(reference_matches, None)
335
336        unnumbered_book_aliases = get_unnumbered_book_aliases(self.book_aliases)
337
338        while reference_match or brackets_match:
339            # Handle the start or end of a bracket
340            # 0 means bracket, 1 means reference
341            values = [
342                brackets_match.start() if brackets_match else math.inf,
343                reference_match.start() if reference_match else math.inf,
344            ]
345
346            book_ref = None
347            index_of_minimum = values.index(min(values))
348
349            if brackets_match and index_of_minimum == 0:
350                if brackets_match.group(0) == "(":
351                    if len(bracket_stack) > 0:
352                        bracket_stack.append(bracket_stack[-1])
353                if brackets_match.group(0) == ")":
354                    if len(bracket_stack) > 0:
355                        del bracket_stack[-1]
356                brackets_match = next(brackets_matches, None)
357
358            if reference_match and index_of_minimum == 1:
359                match_str, book_name, match_with_book, match_without_book = (
360                    reference_match.groups()
361                )
362                try:
363                    if book_name:
364                        respaced_book_name = add_space_after_book_number(
365                            normalize_spacing(trim_trailing_period(book_name)),
366                            unnumbered_book_aliases,
367                            self.language.number_prefixes,
368                        )
369
370                        if respaced_book_name in self.book_aliases:
371                            library_id, book_id = self.book_aliases[respaced_book_name]
372                            book = self.books[library_id, book_id]
373                            last_range = book_reference(
374                                library_id, book_id
375                            ).last_range()
376                            if match_with_book:
377                                if matches := self.match_chapter_range(match_with_book):
378                                    # Rom 1:2-3:4
379                                    book_ref = make_chapter_range(last_range, matches)
380                                    if book_ref is not None or yield_nones:
381                                        yield (match_str, book_ref)
382                                elif matches := self.match_chapter_verses(
383                                    match_with_book
384                                ):
385                                    # Rom 3:4,6-9
386                                    book_ref = self.make_chapter_verses(
387                                        last_range, matches
388                                    )
389                                    if book_ref is not None or yield_nones:
390                                        yield (match_str, book_ref)
391                                elif matches := self.match_number_ranges(
392                                    match_with_book
393                                ):
394                                    if book.chapters == 1:
395                                        # Phlm 3-4 (verse)
396                                        v = verse_range(
397                                            last_range.start.library,
398                                            last_range.start.book,
399                                            1,
400                                            1,
401                                        )
402                                        book_ref = self.make_number_ranges(v, matches)
403                                        if book_ref is not None or yield_nones:
404                                            yield (match_str, book_ref)
405                                    if book.chapters > 1:
406                                        # Rom 3-4 (chapter)
407                                        book_ref = self.make_number_ranges(
408                                            last_range, matches, as_chapters=True
409                                        )
410                                        if book_ref is not None or yield_nones:
411                                            yield (match_str, book_ref)
412                                else:
413                                    yield (match_str, None)
414                            else:  # no associated reference
415                                if bracket_stack:
416                                    bracket_stack[-1] = last_range
417                                else:
418                                    bracket_stack.append(last_range)
419                                if yield_books and (
420                                    trim_trailing_period(match_str)
421                                    not in self.language.ambiguous_aliases
422                                ):
423                                    yield (
424                                        match_str,
425                                        book_reference(library_id, book_id),
426                                    )
427                    elif match_without_book and use_context:
428                        if bracket_stack:
429                            last_range = bracket_stack[-1]
430                            library_id, book_id = (
431                                last_range.start.library,
432                                last_range.start.book,
433                            )
434                            if matches := self.match_chapter_range(match_without_book):
435                                # 1:2-3:4
436                                book_ref = make_chapter_range(last_range, matches)
437                                if book_ref is not None or yield_nones:
438                                    yield (match_without_book, book_ref)
439                            elif matches := self.match_chapter_verses(
440                                match_without_book
441                            ):
442                                # 3:4,6-9
443                                book_ref = self.make_chapter_verses(last_range, matches)
444                                if book_ref is not None or yield_nones:
445                                    yield (match_without_book, book_ref)
446                            elif matches := self.match_number_ranges(
447                                match_without_book
448                            ):
449                                # v.2, vv.3-4
450                                book_ref = self.make_number_ranges(last_range, matches)
451                                if book_ref is not None or yield_nones:
452                                    yield (match_without_book, book_ref)
453                            else:
454                                if yield_nones:
455                                    yield (match_without_book, None)
456                        else:
457                            if yield_nones:
458                                yield (match_without_book, None)
459
460                except ValueError:
461                    if yield_nones:
462                        yield (match_str or match_without_book, None)
463
464                if book_ref:
465                    last_range = book_ref.ranges[-1]
466                    if bracket_stack:
467                        bracket_stack[-1] = last_range
468                    else:
469                        bracket_stack.append(last_range)
470
471                reference_match = next(reference_matches, None)
472
473    def match_chapter_range(self, text) -> Match | None:
474        """Match a pair of chapter-and-verse references.
475
476        Example:
477            `Rom 1:2-3:4`
478        """
479        if match := self.CHAPTER_RANGE_CAPTURE.search(text):
480            return match
481        return None
482
483    def match_chapter_verses(self, text) -> Match | None:
484        """Match a verse list preceded by a chapter marker.
485
486        Example:
487            `Rom 3:4,6-9`
488        """
489        if match := self.CHAPTER_VERSES_CAPTURE.search(text):
490            return match
491        return None
492
493    def match_number_ranges(self, text) -> list[str]:
494        """Match a list of numbers and number ranges.
495
496        Example:
497            `Phlm 1,3-4` (verse), `Rom 1,3-4` (chapter)
498        """
499        return self.RANGE_OR_NUMBER_COMPILED.findall(text)
500
501    def make_number_ranges(
502        self, last: Range, matches: list[str], as_chapters: bool = False
503    ) -> Reference | None:
504        """Create a reference from a list of numbers and number ranges.
505
506        Args:
507            last: a verse to which this number list is relative.
508            matches: the result of `match_number_ranges()`
509            as_chapters: Treat these numbers as chapters rather than verses.
510
511        Example:
512            `Phlm 3-4` (verse), `Rom 3-4` (chapter)
513
514        Note:
515            If the end number is less than the start number, we check whether it
516            might be an abbreviated reference. `123-24` or even `17-8` can be
517            interpreted as `123-124` and `17-18` respectively. We test whether the
518            second number has fewer digits than the first.
519        """
520        ranges = []
521        for text in matches:
522            if range_matches := self.RANGE_CAPTURE.match(text):
523                start, end = range_matches.group(1, 2)
524            elif number_matches := self.NUMBER_CAPTURE.match(text):
525                start = end = number_matches.group(1)
526            else:
527                continue
528            if parse_number(end) < parse_number(start):
529                if new_end := infer_abbreviation(start, end):
530                    end = new_end
531                else:
532                    return None
533            if as_chapters:
534                if last.is_same_book():
535                    ranges.append(
536                        range(
537                            verse(
538                                last.start.library,
539                                last.start.book,
540                                parse_number(start),
541                                1,
542                            ),
543                            verse(
544                                last.end.library, last.end.book, parse_number(end), 999
545                            ),
546                        )
547                    )
548            else:
549                if last.is_same_chapter():
550                    ranges.append(
551                        range(
552                            verse(
553                                last.start.library,
554                                last.start.book,
555                                last.start.chapter,
556                                parse_number(start),
557                            ),
558                            verse(
559                                last.end.library,
560                                last.end.book,
561                                last.end.chapter,
562                                parse_number(end),
563                            ),
564                        )
565                    )
566        if ranges:
567            return reference(*ranges)
568        else:
569            return None
570
571    def make_chapter_verses(self, last: Range, match: Match) -> Reference | None:
572        """Create a reference from verse list preceded by a chapter marker.
573
574        See `match_chapter_verses`.
575
576        Example:
577            `Rom 3:4,6-9`
578        """
579        if last.is_same_book():
580            chapter, numbers = match.groups()
581            if range_match := self.match_number_ranges(numbers):
582                last_range = range(
583                    verse(last.start.library, last.start.book, int(chapter), 1),
584                    verse(last.end.library, last.end.book, int(chapter), 999),
585                )
586                if reference := self.make_number_ranges(last_range, range_match):
587                    return reference
588        return None
589
590
591def make_chapter_range(last: Range, match: Match) -> Reference | None:
592    """Create pair of chapter-and-verse references from a match.
593
594    See `match_chapter_range`.
595
596    Example:
597        `Rom 1:2-3:4`
598    """
599    if last.is_same_book():
600        c1, v1, c2, v2 = match.groups()
601        return reference(
602            range(
603                verse(
604                    last.start.library,
605                    last.start.book,
606                    parse_number(c1),
607                    parse_number(v1),
608                ),
609                verse(
610                    last.end.library, last.end.book, parse_number(c2), parse_number(v2)
611                ),
612            )
613        )
614    return None
615
616
617def escape_book_name(name: str) -> str:
618    """Regexp escape a name, but match multiple spaces.
619
620    This covers line wrapping, indentation, and so on, in the middle of a book
621    name.
622    """
623    return r"\s+".join([re.escape(_) for _ in name.split(" ")])
624
625
626def long_names_first(names: list[str]) -> list[str]:
627    """Remove duplicates and sort by length descending."""
628    return sorted(set(names), key=len)[::-1]
629
630
631def infer_abbreviation(start: str, end: str) -> str | None:
632    """If a number range ends with a smaller number, try to infer an abbreviation.
633
634    If the end number has fewer digits the the start number then we return a
635    copy of the start number with its final digits overprinted by the end
636    number.
637
638    * In `123-24`, `24` clearly means `124`.
639    * In `12-4`, `4` should probably be interpreted as `14`.
640    * In `9-4`, `4` is simply a wrong number. We could guess `14`, but it's too
641        uncertain.
642
643    Returns:
644        None: if no inference is made.
645    """
646    if int(end) < int(start):
647        if len(end) < len(start):
648            return start[: 0 - len(end)] + end
649        else:
650            return None
651    return end