import type { LeakSet, LeakSource } from "#types/leak.types"; import { PACKAGE_FILE, PATH_SEPARATOR, PAYLOAD_ID_KEY, RULE_TAG, SOURCE_HARNESS, SOURCE_INVENTORY, SOURCE_LOCAL_RULES, SOURCE_MEMBERS, SOURCE_PATHS, SOURCE_SCRIPTS, SOURCE_SEEDS, SOURCE_STAGES, SOURCE_SURFACE_RULES, SOURCE_TAXONOMY, SURFACE_RULE_TAG, WORD_SEPARATOR, } from "#configuration/constants/leak.constants"; import { ROOT, absolutePath, paths } from "@ssot/paths"; import { SURFACE_RULES_FOLDER, UTF8 } from "#configuration/constants/source.constants"; import { concernTags, containersFor, folderFor, governedRoots, } from "@ssot/govlab/shared/manifests/taxonomy.manifest.ts"; import { join, relative, sep } from "node:path"; import { readFileSync, readdirSync } from "node:fs"; import { stageLabels, stagesFor } from "@govlab/pipeline/orchestrators/codebase.verify.stages.ts"; import { ANATOMY } from "@banes-lab/web/core/assets/anatomy.generated.ts"; import type { AnatomyFolder } from "@banes-lab/web/types/anatomy.types.ts"; import { HARNESS_TOKENS } from "@ssot/govlab/shared/manifests/vocabulary.manifest.ts"; import type { Inventory } from "#types/inventory.types"; import { ONTOLOGY } from "@banes-lab/web/core/assets/ontology.generated.ts"; import type { SeedReport } from "#types/lesson.types"; import { identifierTokens } from "#core/matchers/leak.matcher"; import { readJsonOrNull } from "#core/persistence/report.persistence"; import { workspaceMembers } from "@govlab/docs/manifest/workspaces.ts"; const isRecord = function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; }; const stemsOf = function stemsOf(folder: string, tag: string): string[] { return readdirSync(folder) .filter((name) => name.endsWith(tag)) .map((name) => name.slice(0, -tag.length)); }; const segmentsOf = function segmentsOf(path: string): string[] { return path.split(PATH_SEPARATOR).filter((segment) => segment.length > 0); }; const relativeSegments = function relativeSegments(absolute: string): string[] { return segmentsOf(relative(ROOT, absolute).split(sep).join(PATH_SEPARATOR)); }; const pathTokens = function pathTokens(node: unknown): string[] { if (typeof node === "string") { return relativeSegments(node); } if (!isRecord(node)) { return []; } return Object.entries(node).flatMap(([key, value]) => [key, ...pathTokens(value)]); }; const readPackage = function readPackage(dir: string): Record | null { const file = join(ROOT, ...segmentsOf(dir), PACKAGE_FILE); const parsed: unknown = JSON.parse(readFileSync(file, UTF8)); return isRecord(parsed) ? parsed : null; }; const scriptNames = function scriptNames(): string[] { const scripts = readPackage("")?.["scripts"]; return isRecord(scripts) ? Object.keys(scripts) : []; }; const memberTokens = function memberTokens(): string[] { return workspaceMembers(ROOT).flatMap((dir) => { const name = readPackage(dir)?.["name"]; const named = typeof name === "string" ? [name, ...name.split(PATH_SEPARATOR)] : []; return [...segmentsOf(dir), ...named]; }); }; const taxonomyTokens = function taxonomyTokens(): string[] { const roots = governedRoots().flatMap((root) => [...segmentsOf(root), ...containersFor(root)]); const folders = concernTags().flatMap((tag) => { const folder = folderFor(tag); return folder === undefined ? [] : [folder]; }); return [...roots, ...folders]; }; const stageTokens = function stageTokens(): string[] { return stageLabels(stagesFor({ cleanCommentsIgnore: "", hexIgnore: "", qualityRoot: "", scope: new Set() })); }; const headOf = function headOf(value: string | null): string[] { const [head] = (value ?? "").split(WORD_SEPARATOR); return head === undefined || head.length === 0 ? [] : [head]; }; const stringsOf = function stringsOf(value: unknown): string[] { if (typeof value === "string") { return [value]; } if (Array.isArray(value)) { return value.flatMap(stringsOf); } return isRecord(value) ? Object.values(value).flatMap(stringsOf) : []; }; const idsOf = function idsOf(value: unknown): string[] { if (Array.isArray(value)) { return value.flatMap(idsOf); } if (!isRecord(value)) { return []; } const own = value[PAYLOAD_ID_KEY]; return [...(typeof own === "string" ? [own] : []), ...Object.values(value).flatMap(idsOf)]; }; const ontologyVocabulary = function ontologyVocabulary(): string[] { return [...idsOf(ONTOLOGY), ...stringsOf(ONTOLOGY).flatMap(identifierTokens)]; }; const treeVocabulary = function treeVocabulary(folder: AnatomyFolder): string[] { const own = [folder.name, ...segmentsOf(folder.path), ...(folder.layer === null ? [] : [folder.layer])]; const files = folder.files.flatMap((file) => [ file.name, ...segmentsOf(file.path), ...file.definitions.map((definition) => definition.name), ]); return [...own, ...files, ...folder.folders.flatMap(treeVocabulary)]; }; export const anatomyVocabulary = function anatomyVocabulary(): string[] { return treeVocabulary(ANATOMY.tree).flatMap((token) => [token, ...identifierTokens(token)]); }; const publishedVocabulary = function publishedVocabulary(inventory: Inventory): Set { const avoidance = inventory.records .filter((record) => record.kind === "avoidance") .flatMap((record) => [record.slug, ...headOf(record.never), ...headOf(record.always)]); return new Set([...ontologyVocabulary(), ...anatomyVocabulary(), ...avoidance]); }; export const deriveLeakSet = function deriveLeakSet(inventory: Inventory, seeds: SeedReport): LeakSet { const groups: [string, string[]][] = [ [SOURCE_INVENTORY, inventory.records.map((record) => record.slug)], [SOURCE_SEEDS, seeds.seeds.map((seed) => seed.id)], [SOURCE_SURFACE_RULES, stemsOf(absolutePath("collaboration", SURFACE_RULES_FOLDER), SURFACE_RULE_TAG)], [SOURCE_PATHS, pathTokens(paths)], [SOURCE_SCRIPTS, scriptNames()], [SOURCE_MEMBERS, memberTokens()], [SOURCE_STAGES, stageTokens()], [SOURCE_LOCAL_RULES, stemsOf(absolutePath("govlabHost.rules"), RULE_TAG)], [SOURCE_TAXONOMY, taxonomyTokens()], [SOURCE_HARNESS, HARNESS_TOKENS.flatMap((token) => [token, token.toUpperCase()])], ]; const published = publishedVocabulary(inventory); const tokens = new Set(); const sources: LeakSource[] = []; for (const [name, found] of groups) { const unique = new Set( found.map((token) => token.trim()).filter((token) => token.length > 0 && !published.has(token)), ); sources.push({ name, tokens: unique.size }); for (const token of unique) { tokens.add(token); } } return { sources, tokens: [...tokens].sort((a, b) => a.localeCompare(b)) }; }; export const leakTarget = function leakTarget(): string { return absolutePath("govlabHost.reports.content.leaks"); }; const isLeakSet = function isLeakSet(value: unknown): value is LeakSet { return isRecord(value) && Array.isArray(value["tokens"]) && Array.isArray(value["sources"]); }; export const readLeakSet = function readLeakSet(): LeakSet | null { const parsed = readJsonOrNull(leakTarget()); return isLeakSet(parsed) ? parsed : null; }; const firstSegments = function firstSegments(locations: readonly string[]): string[] { return locations.flatMap((location) => { const [first] = segmentsOf(location); return first === undefined ? [] : [first]; }); }; export const workspaceRoots = function workspaceRoots(): Set { const declared = pathTokens(paths).filter((token) => token.startsWith("_") || token.startsWith(".")); return new Set([...firstSegments(workspaceMembers(ROOT)), ...firstSegments(governedRoots()), ...declared]); };