# core/pipelines/site.pipeline.ts

> 153 lines of code and 39 definitions.

Tree: Bane's Lab Build Scripts
Language: typescript
Layer: processing
Canonical: https://banes-lab.com/anatomy/build-scripts#file-build-scripts-core-pipelines-site-pipeline-ts
Source text: https://banes-lab.com/source/build-scripts/core/pipelines/site.pipeline.ts.txt

Listed in [core/pipelines](https://banes-lab.com/api/source/build-scripts/core/pipelines.md).

## Definitions

- `wavesOf` (lexical_declaration, line 84, exported)
- `discoverSteps` (lexical_declaration, line 33, exported)
- `runWave` (lexical_declaration, line 137)
- `runPhase` (lexical_declaration, line 162, exported)
- `refuseDuplicateGivers` (lexical_declaration, line 38)
- `refuseMissing` (lexical_declaration, line 55)
- `wavesFrom` (lexical_declaration, line 65)
- `refuseUndeclared` (lexical_declaration, line 93)
- `refuseCachedGives` (lexical_declaration, line 101)
- `runWaves` (lexical_declaration, line 154)
- `finish` (lexical_declaration, line 121)
- `STEP_SUFFIX` (lexical_declaration, line 20)
- `MEGABYTE` (lexical_declaration, line 21)
- `START_PHASE` (lexical_declaration, line 22)
- `LEDGER_NAME` (lexical_declaration, line 23)
- `Waves` (type_alias_declaration, line 25)
- `Finished` (interface_declaration, line 27)
- `givers` (lexical_declaration, line 39)
- `first` (lexical_declaration, line 42)
- `inMode` (lexical_declaration, line 51)
- `missing` (lexical_declaration, line 58)
- `ready` (lexical_declaration, line 69)
- `next` (lexical_declaration, line 73)
- `wave` (lexical_declaration, line 74)
- `earlier` (lexical_declaration, line 86, exported)
- `known` (lexical_declaration, line 87, exported)
- `pending` (lexical_declaration, line 88, exported)
- `declared` (lexical_declaration, line 94)
- `extra` (lexical_declaration, line 95)
- `cacheDecision` (lexical_declaration, line 108, exported)
- `key` (lexical_declaration, line 116, exported)
- `present` (lexical_declaration, line 117, exported)
- `started` (lexical_declaration, line 122)
- `decision` (lexical_declaration, line 123)
- `skipped` (lexical_declaration, line 125)
- `outcome` (lexical_declaration, line 128)
- `ledger` (lexical_declaration, line 142)
- `finished` (lexical_declaration, line 143)
- `given` (lexical_declaration, line 150)

## Contained in

- [core/pipelines](https://banes-lab.com/anatomy/build-scripts/folder-build-scripts-core-pipelines.md)

## Uses

- [configuration/strings/site.strings.ts](https://banes-lab.com/source/build-scripts/configuration/strings/site.strings.ts.md)
- [core/loaders/folder.loader.ts](https://banes-lab.com/source/build-scripts/core/loaders/folder.loader.ts.md)
- [core/plugins/site.plugin.ts](https://banes-lab.com/source/build-scripts/core/plugins/site.plugin.ts.md)
- [core/registries/step.registry.ts](https://banes-lab.com/source/build-scripts/core/registries/step.registry.ts.md)

## Used by

- [core/plugins/site.plugin.ts](https://banes-lab.com/source/build-scripts/core/plugins/site.plugin.ts.md)

## Linked from

- [configuration/strings](https://banes-lab.com/anatomy/build-scripts/folder-build-scripts-configuration-strings.md)
- [core/loaders](https://banes-lab.com/anatomy/build-scripts/folder-build-scripts-core-loaders.md)
- [core/pipelines](https://banes-lab.com/anatomy/build-scripts/folder-build-scripts-core-pipelines.md)
- [core/plugins](https://banes-lab.com/anatomy/build-scripts/folder-build-scripts-core-plugins.md)
- [core/registries](https://banes-lab.com/anatomy/build-scripts/folder-build-scripts-core-registries.md)

## Source

```typescript
import type { CacheDecision, SiteMode, SiteOutcome, SitePhase, SiteState, SiteStep } from "#types/site.types";
import { cacheFile, createFingerprintIndex } from "@govlab/content-fingerprint";
import {
    cachedGive,
    duplicateGiver,
    missingNeed,
    skippedLine,
    stepLine,
    stuckSteps,
    undeclaredGive,
} from "#configuration/strings/site.strings";
import type { FingerprintIndex } from "@govlab/content-fingerprint";
import { absolutePath } from "@ssot/paths";
import { existsSync } from "node:fs";
import { importFolder } from "#core/loaders/folder.loader";
import { performance } from "node:perf_hooks";
import process from "node:process";
import { registeredSteps } from "#core/registries/step.registry";

const STEP_SUFFIX = ".step.ts";
const MEGABYTE = 1_048_576;
const START_PHASE: SitePhase = "start";
const LEDGER_NAME = "site-steps";

type Waves = readonly (readonly SiteStep[])[];

interface Finished {
    readonly milliseconds: number;
    readonly outcome: SiteOutcome;
    readonly step: SiteStep;
}

export const discoverSteps = async function discoverSteps(): Promise<readonly SiteStep[]> {
    await importFolder("app.buildSteps", STEP_SUFFIX);
    return registeredSteps();
};

const refuseDuplicateGivers = function refuseDuplicateGivers(steps: readonly SiteStep[]): void {
    const givers = new Map<string, string>();
    for (const step of steps) {
        for (const key of step.gives) {
            const first = givers.get(key);
            if (first !== undefined) {
                throw new Error(duplicateGiver(key, first, step.name));
            }
            givers.set(key, step.name);
        }
    }
};

const inMode = function inMode(steps: readonly SiteStep[], mode: SiteMode, phase: SitePhase): readonly SiteStep[] {
    return steps.filter((step) => step.phase === phase && step.modes.includes(mode));
};

const refuseMissing = function refuseMissing(steps: readonly SiteStep[], known: ReadonlySet<string>): void {
    const given = new Set([...known, ...steps.flatMap((step) => step.gives)]);
    for (const step of steps) {
        const missing = step.needs.find((key) => !given.has(key));
        if (missing !== undefined) {
            throw new Error(missingNeed(step.name, missing));
        }
    }
};

const wavesFrom = function wavesFrom(pending: readonly SiteStep[], available: ReadonlySet<string>): Waves {
    if (pending.length === 0) {
        return [];
    }
    const ready = pending.filter((step) => step.needs.every((key) => available.has(key)));
    if (ready.length === 0) {
        throw new Error(stuckSteps(pending.map((step) => step.name)));
    }
    const next = new Set([...available, ...ready.flatMap((step) => step.gives)]);
    const wave = ready.toSorted((left, right) => left.name.localeCompare(right.name));
    return [
        wave,
        ...wavesFrom(
            pending.filter((step) => !ready.includes(step)),
            next,
        ),
    ];
};

export const wavesOf = function wavesOf(steps: readonly SiteStep[], phase: SitePhase, mode: SiteMode): Waves {
    refuseDuplicateGivers(steps);
    const earlier = phase === START_PHASE ? [] : inMode(steps, mode, START_PHASE).flatMap((step) => step.gives);
    const known = new Set(earlier);
    const pending = inMode(steps, mode, phase);
    refuseMissing(pending, known);
    return wavesFrom(pending, known);
};

const refuseUndeclared = function refuseUndeclared(step: SiteStep, outcome: SiteOutcome): void {
    const declared = new Set(step.gives);
    const extra = Object.keys(outcome.gives).find((key) => !declared.has(key));
    if (extra !== undefined) {
        throw new Error(undeclaredGive(step.name, extra));
    }
};

const refuseCachedGives = function refuseCachedGives(step: SiteStep, outcome: SiteOutcome): void {
    const given = Object.keys(outcome.gives).at(0);
    if (step.cache !== null && given !== undefined) {
        throw new Error(cachedGive(step.name, given));
    }
};

export const cacheDecision = function cacheDecision(
    step: SiteStep,
    state: SiteState,
    ledger: FingerprintIndex,
): CacheDecision {
    if (step.cache === null) {
        return { key: null, skip: false };
    }
    const key = step.cache.key(state);
    const present = step.cache.outputs.every((output) => existsSync(absolutePath(output)));
    return { key, skip: present && ledger.unchanged(step.name, key) };
};

const finish = async function finish(step: SiteStep, state: SiteState, ledger: FingerprintIndex): Promise<Finished> {
    const started = performance.now();
    const decision = cacheDecision(step, state, ledger);
    if (decision.skip) {
        const skipped = { gives: {}, line: skippedLine(step.name) };
        return { milliseconds: performance.now() - started, outcome: skipped, step };
    }
    const outcome = await step.run(state);
    refuseUndeclared(step, outcome);
    refuseCachedGives(step, outcome);
    if (decision.key !== null) {
        ledger.update(step.name, decision.key);
    }
    return { milliseconds: performance.now() - started, outcome, step };
};

const runWave = async function runWave(
    wave: readonly SiteStep[],
    state: SiteState,
    write: (line: string) => void,
): Promise<SiteState> {
    const ledger = createFingerprintIndex({ file: cacheFile(LEDGER_NAME) });
    const finished = await Promise.all(wave.map(async (step) => finish(step, state, ledger)));
    if (wave.some((step) => step.cache !== null)) {
        ledger.flush();
    }
    finished.forEach((done) => {
        write(done.outcome.line + stepLine(done.step.name, done.milliseconds, process.memoryUsage().rss / MEGABYTE));
    });
    const given = Object.fromEntries(finished.flatMap((done) => Object.entries(done.outcome.gives)));
    return { ...state, ...given };
};

const runWaves = async function runWaves(
    waves: Waves,
    state: SiteState,
    write: (line: string) => void,
): Promise<SiteState> {
    return waves.reduce(async (previous, wave) => runWave(wave, await previous, write), Promise.resolve(state));
};

export const runPhase = async function runPhase(
    steps: readonly SiteStep[],
    phase: SitePhase,
    state: SiteState,
    write: (line: string) => void,
): Promise<SiteState> {
    return runWaves(wavesOf(steps, phase, state.mode), state, write);
};
```