# core/normalizers/word.normalizer.ts

> 109 lines of code and 37 definitions.

Tree: Site tree
Language: typescript
Layer: processing
Canonical: https://banes-lab.com/anatomy/tree#file-core-normalizers-word-normalizer-ts
Source text: https://banes-lab.com/assets/sources/source.49332911fe8b6cc57dc20c220d6b2df6efaccb81ab374c208172fb51e97ee3f4.generated.txt

## Definitions

- `normalizeWord` (lexical_declaration, line 39, exported)
- `wordsOf` (lexical_declaration, line 45, exported)
- `tokensOf` (lexical_declaration, line 104, exported)
- `SPACE` (lexical_declaration, line 10)
- `TAG_OPEN` (lexical_declaration, line 11)
- `TAG_CLOSE` (lexical_declaration, line 12)
- `CLOSING_MARK` (lexical_declaration, line 13)
- `Span` (interface_declaration, line 15)
- `Guard` (interface_declaration, line 20)
- `Step` (interface_declaration, line 25)
- `spellingOf` (lexical_declaration, line 30)
- `mapped` (lexical_declaration, line 31)
- `suffix` (lexical_declaration, line 35)
- `spelled` (lexical_declaration, line 40, exported)
- `plural` (lexical_declaration, line 41, exported)
- `words` (lexical_declaration, line 46, exported)
- `current` (lexical_declaration, line 47, exported)
- `tagOf` (lexical_declaration, line 61)
- `closing` (lexical_declaration, line 62)
- `trimmed` (lexical_declaration, line 63)
- `space` (lexical_declaration, line 64)
- `stepOf` (lexical_declaration, line 68)
- `tag` (lexical_declaration, line 69)
- `closes` (lexical_declaration, line 71)
- `protectedSpans` (lexical_declaration, line 80)
- `cursor` (lexical_declaration, line 82)
- `guard` (lexical_declaration, line 83)
- `open` (lexical_declaration, line 85)
- `close` (lexical_declaration, line 86)
- `step` (lexical_declaration, line 90)
- `isProtected` (lexical_declaration, line 100)
- `spans` (lexical_declaration, line 105, exported)
- `tokens` (lexical_declaration, line 106, exported)
- `start` (lexical_declaration, line 107, exported)
- `at` (lexical_declaration, line 108, exported)
- `char` (lexical_declaration, line 109, exported)
- `inWord` (lexical_declaration, line 110, exported)

## Source

```typescript
import {
    PLURAL_SUFFIX,
    PROTECTED_TAGS,
    SPELLINGS,
    SUFFIX_SPELLINGS,
    WORD_CHARACTERS,
} from "#configuration/constants/vocabulary.constants";
import type { WordToken } from "#types/vocabulary.types";

const SPACE = " ";
const TAG_OPEN = "<";
const TAG_CLOSE = ">";
const CLOSING_MARK = "/";

interface Span {
    readonly end: number;
    readonly start: number;
}

interface Guard {
    readonly name: string;
    readonly start: number;
}

interface Step {
    readonly guard: Guard | null;
    readonly span: Span | null;
}

const spellingOf = function spellingOf(word: string): string {
    const mapped = SPELLINGS.get(word);
    if (mapped !== undefined) {
        return mapped;
    }
    const suffix = SUFFIX_SPELLINGS.find(([british]) => word.endsWith(british));
    return suffix === undefined ? word : word.slice(0, -suffix[0].length) + suffix[1];
};

export const normalizeWord = function normalizeWord(word: string): string {
    const spelled = spellingOf(word.toLowerCase());
    const plural = spelled.length > PLURAL_SUFFIX.length && spelled.endsWith(PLURAL_SUFFIX);
    return plural ? spelled.slice(0, -PLURAL_SUFFIX.length) : spelled;
};

export const wordsOf = function wordsOf(phrase: string): readonly string[] {
    const words: string[] = [];
    let current = "";
    for (const char of phrase.toLowerCase() + SPACE) {
        if (WORD_CHARACTERS.includes(char)) {
            current += char;
            continue;
        }
        if (current.length > 0) {
            words.push(normalizeWord(current));
            current = "";
        }
    }
    return words;
};

const tagOf = function tagOf(body: string): { readonly closing: boolean; readonly name: string } {
    const closing = body.startsWith(CLOSING_MARK);
    const trimmed = closing ? body.slice(1) : body;
    const space = trimmed.indexOf(SPACE);
    return { closing, name: (space === -1 ? trimmed : trimmed.slice(0, space)).toLowerCase() };
};

const stepOf = function stepOf(guard: Guard | null, body: string, open: number, close: number): Step {
    const tag = tagOf(body);
    if (guard !== null) {
        const closes = tag.closing && tag.name === guard.name;
        return closes ? { guard: null, span: { end: close + 1, start: guard.start } } : { guard, span: null };
    }
    if (!tag.closing && PROTECTED_TAGS.has(tag.name)) {
        return { guard: { name: tag.name, start: open }, span: null };
    }
    return { guard: null, span: { end: close + 1, start: open } };
};

const protectedSpans = function protectedSpans(text: string): readonly Span[] {
    const spans: Span[] = [];
    let cursor = 0;
    let guard: Guard | null = null;
    while (cursor < text.length) {
        const open = text.indexOf(TAG_OPEN, cursor);
        const close = open === -1 ? -1 : text.indexOf(TAG_CLOSE, open);
        if (open === -1 || close === -1) {
            break;
        }
        const step = stepOf(guard, text.slice(open + 1, close), open, close);
        if (step.span !== null) {
            spans.push(step.span);
        }
        ({ guard } = step);
        cursor = close + 1;
    }
    return guard === null ? spans : [...spans, { end: text.length, start: guard.start }];
};

const isProtected = function isProtected(spans: readonly Span[], at: number): boolean {
    return spans.some((span) => at >= span.start && at < span.end);
};

export const tokensOf = function tokensOf(text: string): readonly WordToken[] {
    const spans = protectedSpans(text);
    const tokens: WordToken[] = [];
    let start = -1;
    for (let at = 0; at <= text.length; at += 1) {
        const char = at < text.length ? text.charAt(at).toLowerCase() : SPACE;
        const inWord = WORD_CHARACTERS.includes(char) && !isProtected(spans, at);
        if (inWord && start === -1) {
            start = at;
            continue;
        }
        if (!inWord && start !== -1) {
            tokens.push({ end: at, start, word: normalizeWord(text.slice(start, at)) });
            start = -1;
        }
    }
    return tokens;
};
```