import { ANGLE_CLOSE, ANGLE_OPEN, DIGITS, ESCAPE, HASH, HEADING_LANGUAGES, KEYWORDS_BY_LANGUAGE, LETTERS, LINE_BREAK, LINE_COMMENT_BY_LANGUAGE, NONTERMINAL_LANGUAGES, QUOTES, SHOUT_LANGUAGES, WORD_CHARS, } from "#configuration/constants/syntax.constants"; import type { Token, TokenKind } from "#types/syntax.types"; const MIN_SHOUT = 2; interface Scan { readonly at: number; readonly token: Token; } const lineEnd = function lineEnd(code: string, from: number): number { const at = code.indexOf(LINE_BREAK, from); return at === -1 ? code.length : at; }; const runOf = function runOf(code: string, from: number, alphabet: string): number { let at = from; while (at < code.length && alphabet.includes(code.charAt(at))) { at += 1; } return at; }; const stringEnd = function stringEnd(code: string, from: number, quote: string): number { let at = from + 1; while (at < code.length && code.charAt(at) !== quote && code.charAt(at) !== LINE_BREAK) { at += code.charAt(at) === ESCAPE ? 2 : 1; } return Math.min(at + 1, code.length); }; const isShout = function isShout(word: string): boolean { return word.length >= MIN_SHOUT && word === word.toUpperCase() && word !== word.toLowerCase(); }; const isTypeLike = function isTypeLike(word: string): boolean { const head = word.charAt(0); return head !== head.toLowerCase() && !isShout(word); }; const wordKind = function wordKind(word: string, language: string): TokenKind { if (KEYWORDS_BY_LANGUAGE.get(language)?.has(word) === true) { return "keyword"; } if (isShout(word)) { return SHOUT_LANGUAGES.has(language) ? "keyword" : "constant"; } return isTypeLike(word) ? "type" : "plain"; }; const atLineStart = function atLineStart(code: string, at: number): boolean { return at === 0 || code.charAt(at - 1) === LINE_BREAK; }; const scanComment = function scanComment(code: string, at: number, language: string): Scan | null { const marker = LINE_COMMENT_BY_LANGUAGE.get(language); if (marker !== undefined && code.startsWith(marker, at)) { const end = lineEnd(code, at); return { at: end, token: { kind: "comment", text: code.slice(at, end) } }; } if (HEADING_LANGUAGES.has(language) && code.startsWith(HASH, at) && atLineStart(code, at)) { const end = lineEnd(code, at); return { at: end, token: { kind: "heading", text: code.slice(at, end) } }; } return null; }; const scanNonterminal = function scanNonterminal(code: string, at: number, language: string): Scan | null { if (!NONTERMINAL_LANGUAGES.has(language) || code.charAt(at) !== ANGLE_OPEN) { return null; } const close = code.indexOf(ANGLE_CLOSE, at); const end = lineEnd(code, at); if (close === -1 || close > end) { return null; } return { at: close + 1, token: { kind: "type", text: code.slice(at, close + 1) } }; }; const scanToken = function scanToken(code: string, at: number, language: string): Scan { const char = code.charAt(at); const special = scanComment(code, at, language) ?? scanNonterminal(code, at, language); if (special !== null) { return special; } if (QUOTES.includes(char)) { const end = stringEnd(code, at, char); return { at: end, token: { kind: "string", text: code.slice(at, end) } }; } if (DIGITS.includes(char)) { const end = runOf(code, at, DIGITS); return { at: end, token: { kind: "number", text: code.slice(at, end) } }; } if (LETTERS.includes(char)) { const end = runOf(code, at, WORD_CHARS); const word = code.slice(at, end); return { at: end, token: { kind: wordKind(word, language), text: word } }; } return { at: at + 1, token: { kind: "plain", text: char } }; }; export const tokenize = function tokenize(code: string, language: string): readonly Token[] { const tokens: Token[] = []; let at = 0; while (at < code.length) { const { at: next, token } = scanToken(code, at, language); const last = tokens.at(-1); if (last?.kind === "plain" && token.kind === "plain") { tokens[tokens.length - 1] = { kind: "plain", text: last.text + token.text }; } else { tokens.push(token); } at = next; } return tokens; };