import { AMBIGUOUS_PHRASES, COMPOUND_JOINER, GAP_CHARACTERS, PROTECTED_TAGS, WORD_CHARACTERS, } from "#configuration/constants/vocabulary.constants"; import type { PhraseIndex, PhraseMatch, VocabularyEntry } from "#types/vocabulary.types"; import { normalizeWord, wordsOf } from "#core/normalizers/word.normalizer"; const TAG_OPEN = "<"; const TAG_CLOSE = ">"; const CLOSING_MARK = "/"; const SPACE = " "; interface Token { readonly end: number; readonly start: number; readonly word: string; } interface Span { readonly end: number; readonly start: number; } interface Guard { readonly name: string; readonly start: number; } type Candidate = readonly [readonly string[], VocabularyEntry]; const AMBIGUOUS_KEYS: ReadonlySet = new Set( [...AMBIGUOUS_PHRASES].map((phrase) => wordsOf(phrase).join(SPACE)), ); const isAmbiguous = function isAmbiguous(words: readonly string[]): boolean { return AMBIGUOUS_KEYS.has(words.join(SPACE)); }; export const buildPhraseIndex = function buildPhraseIndex(vocabulary: readonly VocabularyEntry[]): PhraseIndex { const byFirstWord = new Map(); for (const entry of vocabulary) { const words = wordsOf(entry.phrase); const [first] = words; if (first === undefined) { continue; } byFirstWord.set(first, [...(byFirstWord.get(first) ?? []), [words, entry]]); } for (const candidates of byFirstWord.values()) { candidates.sort((left, right) => right[0].length - left[0].length); } return { byFirstWord }; }; const tagOf = function tagOf(body: string): { readonly closing: boolean; readonly name: string } { const closing = body.startsWith(CLOSING_MARK); const trimmed = closing ? body.slice(1) : body; const space = trimmed.indexOf(SPACE); return { closing, name: (space === -1 ? trimmed : trimmed.slice(0, space)).toLowerCase() }; }; interface Step { readonly guard: Guard | null; readonly span: Span | null; } const stepOf = function stepOf(guard: Guard | null, body: string, open: number, close: number): Step { const tag = tagOf(body); if (guard !== null) { const closes = tag.closing && tag.name === guard.name; return closes ? { guard: null, span: { end: close + 1, start: guard.start } } : { guard, span: null }; } if (!tag.closing && PROTECTED_TAGS.has(tag.name)) { return { guard: { name: tag.name, start: open }, span: null }; } return { guard: null, span: { end: close + 1, start: open } }; }; const protectedSpans = function protectedSpans(text: string): readonly Span[] { const spans: Span[] = []; let cursor = 0; let guard: Guard | null = null; while (cursor < text.length) { const open = text.indexOf(TAG_OPEN, cursor); const close = open === -1 ? -1 : text.indexOf(TAG_CLOSE, open); if (open === -1 || close === -1) { break; } const step = stepOf(guard, text.slice(open + 1, close), open, close); if (step.span !== null) { spans.push(step.span); } ({ guard } = step); cursor = close + 1; } return guard === null ? spans : [...spans, { end: text.length, start: guard.start }]; }; const isProtected = function isProtected(spans: readonly Span[], at: number): boolean { return spans.some((span) => at >= span.start && at < span.end); }; const tokenize = function tokenize(text: string): readonly Token[] { const spans = protectedSpans(text); const tokens: Token[] = []; let start = -1; for (let at = 0; at <= text.length; at += 1) { const char = at < text.length ? text.charAt(at).toLowerCase() : SPACE; const inWord = WORD_CHARACTERS.includes(char) && !isProtected(spans, at); if (inWord && start === -1) { start = at; continue; } if (!inWord && start !== -1) { tokens.push({ end: at, start, word: normalizeWord(text.slice(start, at)) }); start = -1; } } return tokens; }; const gapIsSoft = function gapIsSoft(text: string, from: number, to: number): boolean { for (let at = from; at < to; at += 1) { if (!GAP_CHARACTERS.has(text.charAt(at))) { return false; } } return true; }; const matchesAt = function matchesAt( text: string, tokens: readonly Token[], at: number, words: readonly string[], ): boolean { for (let offset = 0; offset < words.length; offset += 1) { const token = tokens[at + offset]; const previous = tokens[at + offset - 1]; if (token === undefined || token.word !== words[offset]) { return false; } if (offset > 0 && previous !== undefined && !gapIsSoft(text, previous.end, token.start)) { return false; } } return true; }; const insideCompound = function insideCompound( text: string, tokens: readonly Token[], at: number, length: number, ): boolean { const first = tokens[at]; const last = tokens[at + length - 1]; if (first === undefined || last === undefined) { return false; } return text.charAt(first.start - 1) === COMPOUND_JOINER || text.charAt(last.end) === COMPOUND_JOINER; }; const linkable = function linkable([words, entry]: Candidate, seen: ReadonlySet): boolean { return entry.prose && !isAmbiguous(words) && !seen.has(entry.ref); }; const admits = function admits(text: string, tokens: readonly Token[], at: number, seen: ReadonlySet) { return (candidate: Candidate): boolean => { const [words] = candidate; return ( linkable(candidate, seen) && matchesAt(text, tokens, at, words) && !insideCompound(text, tokens, at, words.length) ); }; }; const hitAt = function hitAt( text: string, tokens: readonly Token[], at: number, index: PhraseIndex, seen: ReadonlySet, ): Candidate | null { const token = tokens[at]; if (token === undefined) { return null; } const candidates = index.byFirstWord.get(token.word) ?? []; return candidates.find(admits(text, tokens, at, seen)) ?? null; }; export const matchPhrases = function matchPhrases( text: string, index: PhraseIndex, seen: Set, ): readonly PhraseMatch[] { const tokens = tokenize(text); const matches: PhraseMatch[] = []; let at = 0; while (at < tokens.length) { const hit = hitAt(text, tokens, at, index, seen); const first = tokens[at]; if (hit === null || first === undefined) { at += 1; continue; } const [words, entry] = hit; const last = tokens[at + words.length - 1] ?? first; matches.push({ end: last.end, entry, start: first.start }); seen.add(entry.ref); at += words.length; } return matches; };