core/coordinators/validation.coordinator.ts

core/coordinators/validation.coordinator.ts is a file in Bane's Lab Build Scripts. 199 lines of code and 34 definitions.

import { CATALOG_FILE_BUDGET, RECORDS_SEGMENT } from "#configuration/constants/catalog.constants";
import {
    CATALOG_SITEMAP_ROUTE,
    FULL_TEXT_ROUTE,
    PAGE_SITEMAP_ROUTE,
    RECORD_SITEMAP_ROUTE,
    SEARCH_KEY_FILE,
    SECTION_SITEMAP_ROUTE,
    SITEMAP_ROUTE,
    SOURCE_SITEMAP_ROUTE,
} from "#configuration/constants/site.constants";
import { EMPTY_MISSING_PAGE, UNREFERENCED_FILE } from "#configuration/strings/validation.strings";
import type { Finding, Seen, ValidationRoute } from "#types/validation.types";
import { buildOutput, bytesOrNull, pageIds, readOrNull, sourcePagePaths } from "#core/loaders/build.loader";
import { buildRoutesOf, pngPixelsOf, readArtifact } from "#core/converters/build.converter";
import {
    catalogFiles,
    grammarFindings,
    literalFindings,
    markdownFormFindings,
    slugFindings,
} from "#core/validators/catalog.grammar.validator";
import {
    checkAlternates,
    checkIndexable,
    checkLinks,
    checkPage,
    checkSearchKey,
    checkSitemap,
    checkText,
} from "#core/validators/site.validator";
import { checkArticleImages, checkDates, checkEncodings, checkSchema } from "#core/validators/metadata.validator";
import { checkAssets, checkDiagrams } from "#core/validators/asset.validator";
import { checkFile, checkPageBytes, checkPageHeadings, isServedUrl } from "#core/validators/build.validator";
import { checkMarkdown, checkPayload } from "#core/validators/payload.validator";
import { closureFindings, lookbackFindings } from "#core/validators/link.validator";
import { closureReport, graphReport, siteIndex } from "#core/resolvers/catalog.resolver";
import { fileForPath, jsonFileOf } from "#core/resolvers/page.resolver";
import { fullTextFileOf, isFullTextRoute } from "#core/selectors/site.selector";
import { jsonPathOf, markdownFileOf, markdownPathOf, sitemapPartFiles } from "#core/resolvers/route.resolver";
import { ANATOMY_PAGE } from "@banes-lab/web/ids/page.ids";
import { Buffer } from "node:buffer";
import { EVIDENCE_ABSENT } from "@banes-lab/content/configuration/constants/evidence.constants.ts";
import { SITE_URL } from "@banes-lab/web/core/assets/link.assets.ts";
import { catalogFindings } from "#core/validators/catalog.validator";
import { checkLinkNames } from "#core/validators/element.validator";
import { citationFindings } from "#core/validators/definition.validator";
import { graphFindings } from "#core/validators/graph.validator";
import { isSourceFilePage } from "#core/predicates/route.predicate";
import { overBudget } from "#configuration/strings/catalog.strings";
import { schemaFindings } from "#core/validators/catalog.kind.validator";
import { securityFindings } from "#core/validators/security.validator";
import { servedPageFindings } from "#core/validators/source.page.validator";
import { unreferencedFiles } from "#core/analyzers/asset.analyzer";

const MISSING_FILE = "404.html";
const ROBOTS_FILE = "robots.txt";
const SITEMAP_FILE = SITEMAP_ROUTE.slice(1);
const PAGE_SITEMAP_FILE = PAGE_SITEMAP_ROUTE.slice(1);
const LLMS_FILE = "llms.txt";
const SLASH = "/";
const RECORD_KIND = "record";

const articlePixelsOf = function articlePixelsOf(url: string): number | null {
    const bytes = url.startsWith(SITE_URL + SLASH) ? bytesOrNull(url.slice(SITE_URL.length + SLASH.length)) : null;
    return bytes === null ? null : pngPixelsOf(bytes);
};

const checkRoute = function checkRoute(route: ValidationRoute, served: ReadonlySet<string>, seen: Seen): Finding[] {
    const file = fileForPath(route.path);
    const address = SITE_URL + route.path;
    const page = checkFile(file, (html) => {
        const artifact = readArtifact(html);
        return [
            ...checkPage(file, artifact, address, seen),
            ...checkIndexable(file, artifact, true),
            ...checkSchema(file, artifact, address),
            ...checkDates(file, artifact, address),
            ...checkArticleImages(file, artifact, articlePixelsOf),
            ...checkEncodings(file, artifact, isServedUrl),
            ...checkAlternates(file, artifact, {
                json: SITE_URL + jsonPathOf(route.page, route.tab),
                markdown: SITE_URL + markdownPathOf(route.path),
            }),
            ...checkLinks(file, artifact, served),
            ...checkLinkNames(file, html),
            ...checkPageBytes(file, html),
            ...checkPageHeadings(file, html),
            ...checkFile(markdownFileOf(route.path), (text) =>
                checkMarkdown(markdownFileOf(route.path), text, artifact.title, address),
            ),
        ];
    });
    const json = jsonFileOf(route.page, route.tab);
    return [
        ...page,
        ...checkFile(json, (raw) => [
            ...checkPayload(json, route.page, route.tab, raw),
            ...checkDiagrams(json, raw),
            ...checkAssets(json, raw),
        ]),
    ];
};

const missingPageFindings = function missingPageFindings(): Finding[] {
    return checkFile(MISSING_FILE, (html) => {
        const artifact = readArtifact(html);
        return [
            ...(artifact.text.length > 0 ? [] : [{ file: MISSING_FILE, message: EMPTY_MISSING_PAGE }]),
            ...checkIndexable(MISSING_FILE, artifact, false),
        ];
    });
};

const fullTextPages = function fullTextPages(routes: readonly ValidationRoute[]): readonly string[] {
    return [...new Set(routes.filter(isFullTextRoute).map((route) => route.page))];
};

const fullTextFindings = function fullTextFindings(routes: readonly ValidationRoute[]): Finding[] {
    const teaching = routes.filter(isFullTextRoute).map((route) => SITE_URL + route.path);
    return [
        ...checkFile(FULL_TEXT_ROUTE, (text) => checkText(FULL_TEXT_ROUTE, text, teaching)),
        ...fullTextPages(routes).flatMap((page) => {
            const file = fullTextFileOf(page);
            const own = routes.filter((route) => route.page === page).map((route) => SITE_URL + route.path);
            return checkFile(file, (text) => {
                const bytes = Buffer.byteLength(text);
                const over =
                    bytes > CATALOG_FILE_BUDGET ? [{ file, message: overBudget(bytes, CATALOG_FILE_BUDGET) }] : [];
                return [...checkText(file, text, own), ...over];
            });
        }),
    ];
};

export const validateDiscovery = async function validateDiscovery(): Promise<Finding[]> {
    const outDir = buildOutput();
    const routes = buildRoutesOf(await pageIds(), (page) => readOrNull(jsonFileOf(page, null)));
    const sources = sourcePagePaths(ANATOMY_PAGE);
    const records = sourcePagePaths(RECORDS_SEGMENT);
    const sections = (await pageIds()).filter((page) => page !== ANATOMY_PAGE).flatMap(sourcePagePaths);
    const served = new Set([
        ...routes.map((route) => route.path),
        ...sources,
        ...records,
        ...sections,
        ...routes.map((route) => markdownPathOf(route.path)),
        SLASH + LLMS_FILE,
        SLASH + FULL_TEXT_ROUTE,
        ...fullTextPages(routes).map((page) => SLASH + fullTextFileOf(page)),
        siteIndex().markdown ?? "",
    ]);
    const addresses = routes.map((route) => SITE_URL + route.path);
    const seen: Seen = { descriptions: new Set<string>(), titles: new Set<string>() };
    const listed = [
        ...addresses,
        ...routes.map((route) => SITE_URL + markdownPathOf(route.path)),
        ...routes.map((route) => SITE_URL + jsonPathOf(route.page, route.tab)),
        SITE_URL + SLASH + FULL_TEXT_ROUTE,
        ...fullTextPages(routes).map((page) => SITE_URL + SLASH + fullTextFileOf(page)),
        SITE_URL + siteIndex().json,
        SITE_URL + (siteIndex().markdown ?? ""),
        SITE_URL + CATALOG_SITEMAP_ROUTE,
        SITE_URL + SOURCE_SITEMAP_ROUTE,
    ];
    const routeFiles = new Set([
        ...routes.map((route) => jsonFileOf(route.page, route.tab)),
        ...routes.map((route) => markdownFileOf(route.path)),
    ]);
    const absent = new Set(
        EVIDENCE_ABSENT.flatMap((entry) => (entry.subject.kind === RECORD_KIND ? [entry.subject.ref] : [])),
    );
    const catalog = catalogFiles(outDir, routeFiles);
    return [
        ...catalogFindings(outDir, SITE_URL, routeFiles),
        ...grammarFindings(catalog),
        ...markdownFormFindings(),
        ...schemaFindings(outDir, catalog.json, new Set(catalog.json)),
        ...slugFindings(outDir, catalog.json),
        ...literalFindings(SITE_URL),
        ...closureFindings(closureReport()),
        ...graphFindings(graphReport()),
        ...lookbackFindings(outDir, SITE_URL, absent),
        ...routes.flatMap((route) => checkRoute(route, served, seen)),
        ...(await servedPageFindings(sources, { listed: isSourceFilePage, served, sitemap: SOURCE_SITEMAP_ROUTE }, seen)),
        ...(await servedPageFindings(records, { listed: () => true, served, sitemap: RECORD_SITEMAP_ROUTE }, seen)),
        ...(await servedPageFindings(sections, { listed: () => true, served, sitemap: SECTION_SITEMAP_ROUTE }, seen)),
        ...(await citationFindings(routes)),
        ...missingPageFindings(),
        ...checkFile(PAGE_SITEMAP_FILE, (xml) => checkSitemap(PAGE_SITEMAP_FILE, xml, addresses)),
        ...checkFile(SITEMAP_FILE, (xml) =>
            checkSitemap(SITEMAP_FILE, xml, [
                SITE_URL + PAGE_SITEMAP_ROUTE,
                ...[SOURCE_SITEMAP_ROUTE, RECORD_SITEMAP_ROUTE, SECTION_SITEMAP_ROUTE].flatMap((route) =>
                    sitemapPartFiles(route, (file) => readOrNull(file) !== null).map((file) => SITE_URL + SLASH + file),
                ),
            ]),
        ),
        ...checkFile(ROBOTS_FILE, (text) => checkText(ROBOTS_FILE, text, [`Sitemap: ${SITE_URL}${SITEMAP_ROUTE}`])),
        ...checkFile(SEARCH_KEY_FILE, (text) => checkSearchKey(SEARCH_KEY_FILE, text)),
        ...securityFindings(SITE_URL, new Date()),
        ...checkFile(LLMS_FILE, (text) => checkText(LLMS_FILE, text, listed)),
        ...fullTextFindings(routes),
        ...unreferencedFiles(outDir).map((file) => ({ file, message: UNREFERENCED_FILE })),
    ];
};