core/loaders/leak.loader.ts

core/loaders/leak.loader.ts is a file in Bane's Lab Content. 173 lines of code and 40 definitions.

import type { LeakSet, LeakSource } from "#types/leak.types";
import {
    PACKAGE_FILE,
    PATH_SEPARATOR,
    PAYLOAD_ID_KEY,
    RULE_TAG,
    SOURCE_HARNESS,
    SOURCE_INVENTORY,
    SOURCE_LOCAL_RULES,
    SOURCE_MEMBERS,
    SOURCE_PATHS,
    SOURCE_SCRIPTS,
    SOURCE_SEEDS,
    SOURCE_STAGES,
    SOURCE_SURFACE_RULES,
    SOURCE_TAXONOMY,
    SURFACE_RULE_TAG,
    WORD_SEPARATOR,
} from "#configuration/constants/leak.constants";
import { ROOT, absolutePath, paths } from "@ssot/paths";
import {
    concernTags,
    containersFor,
    folderFor,
    governedRoots,
} from "@ssot/govlab/shared/manifests/taxonomy.manifest.ts";
import { join, relative, sep } from "node:path";
import { readFileSync, readdirSync } from "node:fs";
import { ANATOMY_SNAPSHOTS } from "@banes-lab/web/core/generated/anatomy.generated.ts";
import type { AnatomyFolder } from "@banes-lab/web/types/anatomy.types.ts";
import { HARNESS_TOKENS } from "@ssot/govlab/shared/manifests/vocabulary.manifest.ts";
import type { Inventory } from "#types/inventory.types";
import { ONTOLOGY } from "@banes-lab/web/core/generated/ontology.generated.ts";
import type { SeedReport } from "#types/lesson.types";
import { UTF8 } from "#configuration/constants/source.constants";
import { identifierTokens } from "#core/matchers/leak.matcher";
import { isRecord } from "#core/selectors/payload.selector";
import { readJsonOrNull } from "#core/persistence/report.persistence";
import { stageLabels } from "@govlab/pipeline/core/converters/stage.converter.ts";
import { stagesFor } from "@govlab/pipeline/core/factories/plan.factory.ts";
import { stringsOf } from "#core/converters/section.converter";
import { workspaceMembers } from "@govlab/docs";

const stemsOf = function stemsOf(folder: string, tag: string): string[] {
    return readdirSync(folder)
        .filter((name) => name.endsWith(tag))
        .map((name) => name.slice(0, -tag.length));
};

const segmentsOf = function segmentsOf(path: string): string[] {
    return path.split(PATH_SEPARATOR).filter((segment) => segment.length > 0);
};

const relativeSegments = function relativeSegments(absolute: string): string[] {
    return segmentsOf(relative(ROOT, absolute).split(sep).join(PATH_SEPARATOR));
};

const pathTokens = function pathTokens(node: unknown): string[] {
    if (typeof node === "string") {
        return relativeSegments(node);
    }
    if (!isRecord(node)) {
        return [];
    }
    return Object.entries(node).flatMap(([key, value]) => [key, ...pathTokens(value)]);
};

const readPackage = function readPackage(dir: string): Record<string, unknown> | null {
    const file = join(ROOT, ...segmentsOf(dir), PACKAGE_FILE);
    const parsed: unknown = JSON.parse(readFileSync(file, UTF8));
    return isRecord(parsed) ? parsed : null;
};

const scriptNames = function scriptNames(): string[] {
    const scripts = readPackage("")?.["scripts"];
    return isRecord(scripts) ? Object.keys(scripts) : [];
};

const memberTokens = function memberTokens(): string[] {
    return workspaceMembers(ROOT).flatMap((dir) => {
        const name = readPackage(dir)?.["name"];
        const named = typeof name === "string" ? [name, ...name.split(PATH_SEPARATOR)] : [];
        return [...segmentsOf(dir), ...named];
    });
};

const taxonomyTokens = function taxonomyTokens(): string[] {
    const roots = governedRoots().flatMap((root) => [...segmentsOf(root), ...containersFor(root)]);
    const folders = concernTags().flatMap((tag) => {
        const folder = folderFor(tag);
        return folder === undefined ? [] : [folder];
    });
    return [...roots, ...folders];
};

const stageTokens = function stageTokens(): string[] {
    return stageLabels(stagesFor({ cleanCommentsIgnore: "", hexIgnore: "", qualityRoot: "", scope: new Set() }));
};

const headOf = function headOf(value: string | null): string[] {
    const [head] = (value ?? "").split(WORD_SEPARATOR);
    return head === undefined || head.length === 0 ? [] : [head];
};

const idsOf = function idsOf(value: unknown): string[] {
    if (Array.isArray(value)) {
        return value.flatMap(idsOf);
    }
    if (!isRecord(value)) {
        return [];
    }
    const own = value[PAYLOAD_ID_KEY];
    return [...(typeof own === "string" ? [own] : []), ...Object.values(value).flatMap(idsOf)];
};

const ontologyVocabulary = function ontologyVocabulary(): string[] {
    return [...idsOf(ONTOLOGY), ...stringsOf(ONTOLOGY).flatMap(identifierTokens)];
};

const treeVocabulary = function treeVocabulary(folder: AnatomyFolder): string[] {
    const own = [folder.name, ...segmentsOf(folder.path), ...(folder.layer === null ? [] : [folder.layer])];
    const files = folder.files.flatMap((file) => [
        file.name,
        ...segmentsOf(file.path),
        ...file.definitions.map((definition) => definition.name),
    ]);
    return [...own, ...files, ...folder.folders.flatMap(treeVocabulary)];
};

export const anatomyVocabulary = function anatomyVocabulary(): string[] {
    return [...ANATOMY_SNAPSHOTS.values()]
        .flatMap((snapshot) => treeVocabulary(snapshot.tree))
        .flatMap((token) => [token, ...identifierTokens(token)]);
};

const publishedVocabulary = function publishedVocabulary(inventory: Inventory): Set<string> {
    const avoidance = inventory.records
        .filter((record) => record.kind === "avoidance")
        .flatMap((record) => [record.slug, ...headOf(record.never), ...headOf(record.always)]);
    return new Set([...ontologyVocabulary(), ...anatomyVocabulary(), ...avoidance]);
};

export const deriveLeakSet = function deriveLeakSet(inventory: Inventory, seeds: SeedReport): LeakSet {
    const groups: [string, string[]][] = [
        [SOURCE_INVENTORY, inventory.records.map((record) => record.slug)],
        [SOURCE_SEEDS, seeds.seeds.map((seed) => seed.id)],
        [SOURCE_SURFACE_RULES, stemsOf(absolutePath("app.coordination", "tools/rules"), SURFACE_RULE_TAG)],
        [SOURCE_PATHS, pathTokens(paths)],
        [SOURCE_SCRIPTS, scriptNames()],
        [SOURCE_MEMBERS, memberTokens()],
        [SOURCE_STAGES, stageTokens()],
        [SOURCE_LOCAL_RULES, stemsOf(absolutePath("govlabHost.rules"), RULE_TAG)],
        [SOURCE_TAXONOMY, taxonomyTokens()],
        [SOURCE_HARNESS, HARNESS_TOKENS.flatMap((token) => [token, token.toUpperCase()])],
    ];
    const published = publishedVocabulary(inventory);
    const tokens = new Set<string>();
    const sources: LeakSource[] = [];
    for (const [name, found] of groups) {
        const unique = new Set(
            found.map((token) => token.trim()).filter((token) => token.length > 0 && !published.has(token)),
        );
        sources.push({ name, tokens: unique.size });
        for (const token of unique) {
            tokens.add(token);
        }
    }
    return { sources, tokens: [...tokens].sort((a, b) => a.localeCompare(b)) };
};

export const leakTarget = function leakTarget(): string {
    return absolutePath("govlabHost.reports.content.leaks");
};

const isLeakSet = function isLeakSet(value: unknown): value is LeakSet {
    return isRecord(value) && Array.isArray(value["tokens"]) && Array.isArray(value["sources"]);
};

export const readLeakSet = function readLeakSet(): LeakSet | null {
    const parsed = readJsonOrNull(leakTarget());
    return isLeakSet(parsed) ? parsed : null;
};

const firstSegments = function firstSegments(locations: readonly string[]): string[] {
    return locations.flatMap((location) => {
        const [first] = segmentsOf(location);
        return first === undefined ? [] : [first];
    });
};

export const workspaceRoots = function workspaceRoots(): Set<string> {
    const declared = pathTokens(paths).filter((token) => token.startsWith("_") || token.startsWith("."));
    return new Set([...firstSegments(workspaceMembers(ROOT)), ...firstSegments(governedRoots()), ...declared]);
};