core/matchers/vocabulary.matcher.ts

core/matchers/vocabulary.matcher.ts is a file in Bane's Lab Site. 142 lines of code and 32 definitions.

import {
    AMBIGUOUS_PHRASES,
    COMPOUND_JOINER,
    GAP_CHARACTERS,
    WORD_CHARACTERS,
} from "#configuration/constants/vocabulary.constants";
import type { PhraseCandidate, PhraseIndex, PhraseMatch, VocabularyEntry, WordToken } from "#types/vocabulary.types";
import { tokensOf, wordsOf } from "#core/normalizers/word.normalizer";
import { SPACE } from "#configuration/constants/document.constants";

const COMMA = ",";

const commaJointsOf = function commaJointsOf(phrase: string): readonly boolean[] {
    const joints: boolean[] = [];
    let inWord = false;
    let seenWord = false;
    let comma = false;
    for (const char of phrase.toLowerCase()) {
        if (WORD_CHARACTERS.includes(char)) {
            if (!inWord && seenWord) {
                joints.push(comma);
            }
            inWord = true;
            seenWord = true;
            comma = false;
            continue;
        }
        inWord = false;
        comma ||= char === COMMA;
    }
    return joints;
};

const AMBIGUOUS_KEYS: ReadonlySet<string> = new Set(
    [...AMBIGUOUS_PHRASES].map((phrase) => wordsOf(phrase).join(SPACE)),
);

const isAmbiguous = function isAmbiguous(words: readonly string[]): boolean {
    return AMBIGUOUS_KEYS.has(words.join(SPACE));
};

export const buildPhraseIndex = function buildPhraseIndex(vocabulary: readonly VocabularyEntry[]): PhraseIndex {
    const byFirstWord = new Map<string, PhraseCandidate[]>();
    for (const entry of vocabulary) {
        const words = wordsOf(entry.phrase);
        const [first] = words;
        if (first === undefined) {
            continue;
        }
        byFirstWord.set(first, [...(byFirstWord.get(first) ?? []), [words, entry, commaJointsOf(entry.phrase)]]);
    }
    for (const candidates of byFirstWord.values()) {
        candidates.sort((left, right) => right[0].length - left[0].length);
    }
    return { byFirstWord };
};

const gapIsSoft = function gapIsSoft(text: string, from: number, to: number, comma: boolean): boolean {
    for (let at = from; at < to; at += 1) {
        const char = text.charAt(at);
        if (!GAP_CHARACTERS.has(char) && !(comma && char === COMMA)) {
            return false;
        }
    }
    return true;
};

const matchesAt = function matchesAt(
    text: string,
    tokens: readonly WordToken[],
    at: number,
    [words, , joints]: PhraseCandidate,
): boolean {
    for (let offset = 0; offset < words.length; offset += 1) {
        const token = tokens[at + offset];
        const previous = tokens[at + offset - 1];
        if (token === undefined || token.word !== words[offset]) {
            return false;
        }
        const comma = joints[offset - 1] === true;
        if (offset > 0 && previous !== undefined && !gapIsSoft(text, previous.end, token.start, comma)) {
            return false;
        }
    }
    return true;
};

const insideCompound = function insideCompound(
    text: string,
    tokens: readonly WordToken[],
    at: number,
    length: number,
): boolean {
    const first = tokens[at];
    const last = tokens[at + length - 1];
    if (first === undefined || last === undefined) {
        return false;
    }
    return text.charAt(first.start - 1) === COMPOUND_JOINER || text.charAt(last.end) === COMPOUND_JOINER;
};

const linkable = function linkable([words, entry]: PhraseCandidate, seen: ReadonlySet<string>): boolean {
    return entry.prose && !isAmbiguous(words) && !seen.has(entry.ref);
};

const admits = function admits(text: string, tokens: readonly WordToken[], at: number, seen: ReadonlySet<string>) {
    return (candidate: PhraseCandidate): boolean => {
        const [words] = candidate;
        return (
            linkable(candidate, seen) &&
            matchesAt(text, tokens, at, candidate) &&
            !insideCompound(text, tokens, at, words.length)
        );
    };
};

const hitAt = function hitAt(
    text: string,
    tokens: readonly WordToken[],
    at: number,
    index: PhraseIndex,
    seen: ReadonlySet<string>,
): PhraseCandidate | null {
    const token = tokens[at];
    if (token === undefined) {
        return null;
    }
    const candidates = index.byFirstWord.get(token.word) ?? [];
    return candidates.find(admits(text, tokens, at, seen)) ?? null;
};

export const matchPhrases = function matchPhrases(
    text: string,
    index: PhraseIndex,
    seen: Set<string>,
): readonly PhraseMatch[] {
    const tokens = tokensOf(text);
    const matches: PhraseMatch[] = [];
    let at = 0;
    while (at < tokens.length) {
        const hit = hitAt(text, tokens, at, index, seen);
        const first = tokens[at];
        if (hit === null || first === undefined) {
            at += 1;
            continue;
        }
        const [words, entry] = hit;
        const last = tokens[at + words.length - 1] ?? first;
        matches.push({ end: last.end, entry, start: first.start });
        seen.add(entry.ref);
        at += words.length;
    }
    return matches;
};