configuration/constants/site.constants.ts

configuration/constants/site.constants.ts is a file in Bane's Lab Build Scripts. 175 lines of code and 43 definitions.

import { ANATOMY_PAGE, ONTOLOGY_PAGE } from "@banes-lab/web/ids/page.ids";
import { API_SEGMENT, QUERY_ENDPOINT, RECORDS_SEGMENT } from "#configuration/constants/catalog.constants";
import type { CrawlRule } from "#types/site.types";

export const JSON_ROUTE = "/json/";
export const FILE_ROUTES: readonly string[] = ["/assets/", "/static/"];
export const FULL_TEXT_ROUTE = "llms-full.txt";
export const FULL_TEXT_FOLDER = "llms-full";
export const FULL_TEXT_EXTENSION = ".txt";
export const REFERENCE_PAGES: ReadonlySet<string> = new Set([ONTOLOGY_PAGE, ANATOMY_PAGE]);
export const SITEMAP_ROUTE = "/sitemap.xml";
export const CATALOG_SITEMAP_ROUTE = "/sitemap-catalog.xml";
export const PAGE_SITEMAP_ROUTE = "/sitemap-pages.xml";
export const SOURCE_SITEMAP_ROUTE = "/sitemap-sources.xml";
export const RECORD_SITEMAP_ROUTE = "/sitemap-records.xml";
export const SECTION_SITEMAP_ROUTE = "/sitemap-sections.xml";
export const PAGE_BYTE_BUDGET = 125_000;
export const SOURCE_PART_CHARS = 90_000;
export const SECTION_PART_WEIGHT = 50_000;
export const SITEMAP_PART_PREFIX = "sitemap-";
export const SITEMAP_EXTENSION = ".xml";
export const SITEMAP_PART_JOIN = "-";
export const SITEMAP_URL_CAP = 50_000;
export const SITEMAP_BYTE_CAP = 52_428_800;
export const MARKDOWN_EXTENSION = ".md";
export const SEARCH_KEY_FILE = "indexnow-key.txt";
export const SECURITY_TXT_FILE = ".well-known/security.txt";
export const SECURITY_TXT_DAYS = 364;
export const SECURITY_TXT_LANGUAGES = "en";
export const SECURITY_KEY_FILE = ".well-known/pgp-key.txt";
export const SECURITY_TXT_FIELDS = {
    canonical: "Canonical",
    contact: "Contact",
    encryption: "Encryption",
    expires: "Expires",
    languages: "Preferred-Languages",
} as const;
export const SEARCH_KEY_LENGTH = { max: 128, min: 8 } as const;
const ALPHABET_LENGTH = 26;
const DIGIT_COUNT = 10;

const charactersFrom = function charactersFrom(first: string, count: number): readonly string[] {
    const start = first.codePointAt(0) ?? 0;
    return Array.from({ length: count }, (_, offset) => String.fromCodePoint(start + offset));
};

export const SEARCH_KEY_CHARACTERS: ReadonlySet<string> = new Set([
    ...charactersFrom("a", ALPHABET_LENGTH),
    ...charactersFrom("A", ALPHABET_LENGTH),
    ...charactersFrom("0", DIGIT_COUNT),
    "-",
]);

export const SCHEMA_FIELDS: ReadonlyMap<string, readonly string[]> = new Map([
    ["Dataset", ["name", "description", "creator", "license"]],
    ["PostalAddress", ["addressCountry", "addressLocality", "postalCode", "streetAddress"]],
    ["TechArticle", ["headline", "image", "author", "datePublished", "dateModified"]],
]);

export const PAGE_DATE_FIELDS: readonly string[] = ["datePublished", "dateModified"];

export const ARTICLE_MIN_PIXELS = 50_000;

export const SCHEMA_KEYWORDS: ReadonlySet<string> = new Set(["@id", "@type"]);

export const WEB_PAGE_TYPES: ReadonlySet<string> = new Set(["ProfilePage", "WebPage"]);

const WEB_PAGE_PROPERTIES: ReadonlySet<string> = new Set([
    "author",
    "breadcrumb",
    "dateModified",
    "datePublished",
    "description",
    "encoding",
    "hasPart",
    "inLanguage",
    "isPartOf",
    "mainEntity",
    "name",
    "publisher",
    "url",
]);

export const SCHEMA_PROPERTIES: ReadonlyMap<string, ReadonlySet<string>> = new Map([
    ["BreadcrumbList", new Set(["itemListElement"])],
    ["ContactPoint", new Set(["email", "url"])],
    [
        "CreativeWork",
        new Set(["alternateName", "author", "creator", "description", "license", "name", "sameAs", "url", "version"]),
    ],
    ["DataCatalog", new Set(["description", "name", "publisher", "url"])],
    ["DataDownload", new Set(["contentUrl", "encodingFormat", "name"])],
    ["DefinedTerm", new Set(["description", "inDefinedTermSet", "name", "termCode", "url"])],
    ["DefinedTermSet", new Set(["description", "hasDefinedTerm", "name", "url"])],
    [
        "Dataset",
        new Set([
            "creator",
            "dateModified",
            "description",
            "distribution",
            "identifier",
            "includedInDataCatalog",
            "isAccessibleForFree",
            "keywords",
            "license",
            "name",
            "sameAs",
            "url",
            "version",
        ]),
    ],
    ["ItemList", new Set(["itemListElement", "name", "numberOfItems"])],
    ["ListItem", new Set(["item", "name", "position"])],
    ["MediaObject", new Set(["contentUrl", "encodingFormat"])],
    [
        "Organization",
        new Set([
            "address",
            "contactPoint",
            "description",
            "email",
            "founder",
            "foundingDate",
            "iso6523Code",
            "legalName",
            "logo",
            "name",
            "numberOfEmployees",
            "sameAs",
            "taxID",
            "url",
            "vatID",
        ]),
    ],
    ["Person", new Set(["alternateName", "description", "email", "name", "sameAs", "url", "worksFor"])],
    ["PostalAddress", new Set(["addressCountry", "addressLocality", "postalCode", "streetAddress"])],
    ["QuantitativeValue", new Set(["value"])],
    ["SoftwareSourceCode", new Set(["codeRepository", "isPartOf", "license", "name", "programmingLanguage"])],
    [
        "TechArticle",
        new Set([
            "author",
            "dateModified",
            "datePublished",
            "headline",
            "image",
            "inLanguage",
            "publisher",
        ]),
    ],
    ["ProfilePage", WEB_PAGE_PROPERTIES],
    ["WebPage", WEB_PAGE_PROPERTIES],
    ["WebPageElement", new Set(["encoding", "name", "url"])],
    ["WebSite", new Set(["description", "inLanguage", "name", "publisher", "url"])],
]);

export const SEARCH_AGENTS: readonly string[] = ["Googlebot", "Bingbot"];

export const SEARCH_CRAWL_RULES: readonly CrawlRule[] = [
    { directive: "Disallow", path: `/*${MARKDOWN_EXTENSION}$` },
    { directive: "Disallow", path: `/${API_SEGMENT}/` },
    { directive: "Disallow", path: JSON_ROUTE },
    { directive: "Allow", path: `${JSON_ROUTE}${API_SEGMENT}/${RECORDS_SEGMENT}/` },
    { directive: "Disallow", path: QUERY_ENDPOINT },
    { directive: "Disallow", path: `/${FULL_TEXT_FOLDER}` },
];

export const AI_AGENTS: readonly string[] = [
    "GPTBot",
    "OAI-SearchBot",
    "ChatGPT-User",
    "ClaudeBot",
    "Claude-SearchBot",
    "Claude-User",
    "anthropic-ai",
    "Google-Extended",
    "PerplexityBot",
    "Perplexity-User",
    "CCBot",
    "Applebot-Extended",
    "Bytespider",
    "meta-externalagent",
    "Amazonbot",
    "cohere-ai",
    "DuckAssistBot",
    "YouBot",
];