# core/analyzers/syntax.analyzer.ts

> 118 lines of code and 24 definitions.

Tree: Site tree
Language: typescript
Layer: processing
Canonical: https://banes-lab.com/anatomy/tree#file-core-analyzers-syntax-analyzer-ts
Source text: https://banes-lab.com/assets/sources/source.a263d0e65ee4f660882e4e4086389f0dc25ae6da59bfca9042efd29afbea033a.generated.txt

## Definitions

- `isShout` (lexical_declaration, line 47)
- `wordKind` (lexical_declaration, line 56)
- `isTypeLike` (lexical_declaration, line 51)
- `atLineStart` (lexical_declaration, line 66)
- `tokenize` (lexical_declaration, line 117, exported)
- `scanComment` (lexical_declaration, line 70)
- `scanToken` (lexical_declaration, line 95)
- `MIN_SHOUT` (lexical_declaration, line 19)
- `Scan` (interface_declaration, line 21)
- `lineEnd` (lexical_declaration, line 26)
- `runOf` (lexical_declaration, line 31)
- `stringEnd` (lexical_declaration, line 39)
- `head` (lexical_declaration, line 52)
- `marker` (lexical_declaration, line 71)
- `scanNonterminal` (lexical_declaration, line 83)
- `close` (lexical_declaration, line 87)
- `char` (lexical_declaration, line 96)
- `special` (lexical_declaration, line 97)
- `end` (lexical_declaration, line 110)
- `word` (lexical_declaration, line 111)
- `tokens` (lexical_declaration, line 118, exported)
- `at` (lexical_declaration, line 119, exported)
- `{ at: next, token }` (lexical_declaration, line 121, exported)
- `last` (lexical_declaration, line 122, exported)

## Used by

- [presentation/renderers/syntax.renderer.ts](https://banes-lab.com/source/tree/presentation/renderers/syntax.renderer.ts.md)

## Source

```typescript
import {
    ANGLE_CLOSE,
    ANGLE_OPEN,
    DIGITS,
    ESCAPE,
    HASH,
    HEADING_LANGUAGES,
    KEYWORDS_BY_LANGUAGE,
    LETTERS,
    LINE_BREAK,
    LINE_COMMENT_BY_LANGUAGE,
    NONTERMINAL_LANGUAGES,
    QUOTES,
    SHOUT_LANGUAGES,
    WORD_CHARS,
} from "#configuration/constants/syntax.constants";
import type { Token, TokenKind } from "#types/syntax.types";

const MIN_SHOUT = 2;

interface Scan {
    readonly at: number;
    readonly token: Token;
}

const lineEnd = function lineEnd(code: string, from: number): number {
    const at = code.indexOf(LINE_BREAK, from);
    return at === -1 ? code.length : at;
};

const runOf = function runOf(code: string, from: number, alphabet: string): number {
    let at = from;
    while (at < code.length && alphabet.includes(code.charAt(at))) {
        at += 1;
    }
    return at;
};

const stringEnd = function stringEnd(code: string, from: number, quote: string): number {
    let at = from + 1;
    while (at < code.length && code.charAt(at) !== quote && code.charAt(at) !== LINE_BREAK) {
        at += code.charAt(at) === ESCAPE ? 2 : 1;
    }
    return Math.min(at + 1, code.length);
};

const isShout = function isShout(word: string): boolean {
    return word.length >= MIN_SHOUT && word === word.toUpperCase() && word !== word.toLowerCase();
};

const isTypeLike = function isTypeLike(word: string): boolean {
    const head = word.charAt(0);
    return head !== head.toLowerCase() && !isShout(word);
};

const wordKind = function wordKind(word: string, language: string): TokenKind {
    if (KEYWORDS_BY_LANGUAGE.get(language)?.has(word) === true) {
        return "keyword";
    }
    if (isShout(word)) {
        return SHOUT_LANGUAGES.has(language) ? "keyword" : "constant";
    }
    return isTypeLike(word) ? "type" : "plain";
};

const atLineStart = function atLineStart(code: string, at: number): boolean {
    return at === 0 || code.charAt(at - 1) === LINE_BREAK;
};

const scanComment = function scanComment(code: string, at: number, language: string): Scan | null {
    const marker = LINE_COMMENT_BY_LANGUAGE.get(language);
    if (marker !== undefined && code.startsWith(marker, at)) {
        const end = lineEnd(code, at);
        return { at: end, token: { kind: "comment", text: code.slice(at, end) } };
    }
    if (HEADING_LANGUAGES.has(language) && code.startsWith(HASH, at) && atLineStart(code, at)) {
        const end = lineEnd(code, at);
        return { at: end, token: { kind: "heading", text: code.slice(at, end) } };
    }
    return null;
};

const scanNonterminal = function scanNonterminal(code: string, at: number, language: string): Scan | null {
    if (!NONTERMINAL_LANGUAGES.has(language) || code.charAt(at) !== ANGLE_OPEN) {
        return null;
    }
    const close = code.indexOf(ANGLE_CLOSE, at);
    const end = lineEnd(code, at);
    if (close === -1 || close > end) {
        return null;
    }
    return { at: close + 1, token: { kind: "type", text: code.slice(at, close + 1) } };
};

const scanToken = function scanToken(code: string, at: number, language: string): Scan {
    const char = code.charAt(at);
    const special = scanComment(code, at, language) ?? scanNonterminal(code, at, language);
    if (special !== null) {
        return special;
    }
    if (QUOTES.includes(char)) {
        const end = stringEnd(code, at, char);
        return { at: end, token: { kind: "string", text: code.slice(at, end) } };
    }
    if (DIGITS.includes(char)) {
        const end = runOf(code, at, DIGITS);
        return { at: end, token: { kind: "number", text: code.slice(at, end) } };
    }
    if (LETTERS.includes(char)) {
        const end = runOf(code, at, WORD_CHARS);
        const word = code.slice(at, end);
        return { at: end, token: { kind: wordKind(word, language), text: word } };
    }
    return { at: at + 1, token: { kind: "plain", text: char } };
};

export const tokenize = function tokenize(code: string, language: string): readonly Token[] {
    const tokens: Token[] = [];
    let at = 0;
    while (at < code.length) {
        const { at: next, token } = scanToken(code, at, language);
        const last = tokens.at(-1);
        if (last?.kind === "plain" && token.kind === "plain") {
            tokens[tokens.length - 1] = { kind: "plain", text: last.text + token.text };
        } else {
            tokens.push(token);
        }
        at = next;
    }
    return tokens;
};
```