core/analyzers/syntax.analyzer.ts

core/analyzers/syntax.analyzer.ts is a file in Bane's Lab Site. 118 lines of code and 24 definitions.

import {
    ANGLE_CLOSE,
    ANGLE_OPEN,
    DIGITS,
    ESCAPE,
    HASH,
    HEADING_LANGUAGES,
    KEYWORDS_BY_LANGUAGE,
    LETTERS,
    LINE_COMMENT_BY_LANGUAGE,
    NONTERMINAL_LANGUAGES,
    QUOTES,
    SHOUT_LANGUAGES,
    WORD_CHARS,
} from "#configuration/constants/syntax.constants";
import type { Token, TokenKind } from "#types/syntax.types";
import { LINE_BREAK } from "#configuration/constants/code.constants";

const MIN_SHOUT = 2;

interface Scan {
    readonly at: number;
    readonly token: Token;
}

const lineEnd = function lineEnd(code: string, from: number): number {
    const at = code.indexOf(LINE_BREAK, from);
    return at === -1 ? code.length : at;
};

const runOf = function runOf(code: string, from: number, alphabet: string): number {
    let at = from;
    while (at < code.length && alphabet.includes(code.charAt(at))) {
        at += 1;
    }
    return at;
};

const stringEnd = function stringEnd(code: string, from: number, quote: string): number {
    let at = from + 1;
    while (at < code.length && code.charAt(at) !== quote && code.charAt(at) !== LINE_BREAK) {
        at += code.charAt(at) === ESCAPE ? 2 : 1;
    }
    return Math.min(at + 1, code.length);
};

const isShout = function isShout(word: string): boolean {
    return word.length >= MIN_SHOUT && word === word.toUpperCase() && word !== word.toLowerCase();
};

const isTypeLike = function isTypeLike(word: string): boolean {
    const head = word.charAt(0);
    return head !== head.toLowerCase() && !isShout(word);
};

const wordKind = function wordKind(word: string, language: string): TokenKind {
    if (KEYWORDS_BY_LANGUAGE.get(language)?.has(word) === true) {
        return "keyword";
    }
    if (isShout(word)) {
        return SHOUT_LANGUAGES.has(language) ? "keyword" : "constant";
    }
    return isTypeLike(word) ? "type" : "plain";
};

const atLineStart = function atLineStart(code: string, at: number): boolean {
    return at === 0 || code.charAt(at - 1) === LINE_BREAK;
};

const scanComment = function scanComment(code: string, at: number, language: string): Scan | null {
    const marker = LINE_COMMENT_BY_LANGUAGE.get(language);
    if (marker !== undefined && code.startsWith(marker, at)) {
        const end = lineEnd(code, at);
        return { at: end, token: { kind: "comment", text: code.slice(at, end) } };
    }
    if (HEADING_LANGUAGES.has(language) && code.startsWith(HASH, at) && atLineStart(code, at)) {
        const end = lineEnd(code, at);
        return { at: end, token: { kind: "heading", text: code.slice(at, end) } };
    }
    return null;
};

const scanNonterminal = function scanNonterminal(code: string, at: number, language: string): Scan | null {
    if (!NONTERMINAL_LANGUAGES.has(language) || code.charAt(at) !== ANGLE_OPEN) {
        return null;
    }
    const close = code.indexOf(ANGLE_CLOSE, at);
    const end = lineEnd(code, at);
    if (close === -1 || close > end) {
        return null;
    }
    return { at: close + 1, token: { kind: "type", text: code.slice(at, close + 1) } };
};

const scanToken = function scanToken(code: string, at: number, language: string): Scan {
    const char = code.charAt(at);
    const special = scanComment(code, at, language) ?? scanNonterminal(code, at, language);
    if (special !== null) {
        return special;
    }
    if (QUOTES.includes(char)) {
        const end = stringEnd(code, at, char);
        return { at: end, token: { kind: "string", text: code.slice(at, end) } };
    }
    if (DIGITS.includes(char)) {
        const end = runOf(code, at, DIGITS);
        return { at: end, token: { kind: "number", text: code.slice(at, end) } };
    }
    if (LETTERS.includes(char)) {
        const end = runOf(code, at, WORD_CHARS);
        const word = code.slice(at, end);
        return { at: end, token: { kind: wordKind(word, language), text: word } };
    }
    return { at: at + 1, token: { kind: "plain", text: char } };
};

export const tokenize = function tokenize(code: string, language: string): readonly Token[] {
    const tokens: Token[] = [];
    let at = 0;
    while (at < code.length) {
        const { at: next, token } = scanToken(code, at, language);
        const last = tokens.at(-1);
        if (last?.kind === "plain" && token.kind === "plain") {
            tokens[tokens.length - 1] = { kind: "plain", text: last.text + token.text };
        } else {
            tokens.push(token);
        }
        at = next;
    }
    return tokens;
};