import type { CatalogStore, DataRenderer, Entry, Leaf } from "#types/catalog.types"; import { duplicateAddress, duplicateRef } from "#configuration/strings/catalog.strings"; import { Buffer } from "node:buffer"; import { digestOf } from "@govlab/content-fingerprint"; const JSON_INDENT = 2; const LINE_END = "\n"; export const serialize = function serialize(data: object): string { return JSON.stringify(data, null, JSON_INDENT) + LINE_END; }; export const packedParts = function packedParts( entries: readonly T[], budget: number, ): readonly (readonly T[])[] { const parts: T[][] = []; let current: T[] = []; let bytes = 0; for (const entry of entries) { const size = Buffer.byteLength(serialize(entry)); if (current.length > 0 && bytes + size > budget) { parts.push(current); current = []; bytes = 0; } current.push(entry); bytes += size; } return current.length === 0 ? parts : [...parts, current]; }; const groupBy = function groupBy( entries: readonly (readonly [string, T])[], length: number, ): Map { const groups = new Map(); for (const entry of entries) { const prefix = entry[0].slice(0, length); groups.set(prefix, [...(groups.get(prefix) ?? []), entry]); } return groups; }; const splitGroup = function splitGroup( prefix: string, entries: readonly (readonly [string, T])[], budget: number, ): readonly (readonly [string, readonly (readonly [string, T])[]])[] { const fits = Buffer.byteLength(serialize(Object.fromEntries(entries))) <= budget; const longer = entries.filter(([key]) => key.length > prefix.length); if (fits || longer.length === 0) { return [[prefix, entries]]; } const own = entries.filter(([key]) => key.length <= prefix.length); const deeper = [...groupBy(longer, prefix.length + 1).entries()].flatMap(([next, members]) => splitGroup(next, members, budget), ); return own.length === 0 ? deeper : [[prefix, own], ...deeper]; }; export const prefixShards = function prefixShards( entries: readonly (readonly [string, T])[], budget: number, ): ReadonlyMap { const nonEmpty = entries.filter(([key]) => key.length > 0); return new Map( [...groupBy(nonEmpty, 1).entries()] .flatMap(([prefix, members]) => splitGroup(prefix, members, budget)) .toSorted(([left], [right]) => left.localeCompare(right)), ); }; export const createStore = function createStore(site: string, render: DataRenderer): CatalogStore { const files = new Map(); const entries = new Map(); const put = function put(address: string, body: string): void { if (files.has(address)) { throw new Error(duplicateAddress(address)); } files.set(address, body); }; const addOne = function addOne(leaf: Leaf): Entry { const { identity } = leaf; if (entries.has(identity.ref)) { throw new Error(duplicateRef(identity.ref)); } const json = serialize(leaf.data); put(identity.address.json, json); const { markdown } = identity.address; if (markdown !== null) { put(markdown, leaf.markdown ?? render(identity, leaf.data, site)); } if (leaf.text !== undefined) { put(leaf.text.address, leaf.text.body); } const entry: Entry = { bytes: Buffer.byteLength(json), fingerprint: digestOf(json), href: identity.href === null ? null : site + identity.href, json: site + identity.address.json, kind: identity.kind, markdown: markdown === null ? null : site + markdown, ref: identity.ref, summary: identity.summary, title: identity.title, }; entries.set(identity.ref, entry); return entry; }; return { add: (leaves) => leaves.map(addOne), entries, files }; };