340 lines
14 KiB
JavaScript
340 lines
14 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { createHash, webcrypto } from "node:crypto";
|
|
import { readFileSync, readdirSync, statSync } from "node:fs";
|
|
import path from "node:path";
|
|
import test from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import {
|
|
DataContractError,
|
|
REQUIRED_ASSETS,
|
|
SCHEMA_VERSION,
|
|
loadDashboardData,
|
|
validateAssetSet,
|
|
} from "../js/data.js";
|
|
import { renderUtahCoverage } from "../js/charts.js";
|
|
|
|
const DASHBOARD_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const PUBLIC_DATA = path.join(DASHBOARD_ROOT, "public", "data");
|
|
|
|
function json(filename) {
|
|
return JSON.parse(readFileSync(path.join(PUBLIC_DATA, filename), "utf8"));
|
|
}
|
|
|
|
function assetSet() {
|
|
return Object.fromEntries(
|
|
Object.entries(REQUIRED_ASSETS).map(([name, filename]) => [name, json(filename)]),
|
|
);
|
|
}
|
|
|
|
function publicDataFetch(overrides = new Map()) {
|
|
return async (requestUrl) => {
|
|
const filename = path.basename(new URL(String(requestUrl)).pathname);
|
|
const source = path.join(PUBLIC_DATA, filename);
|
|
let bytes;
|
|
try {
|
|
bytes = overrides.has(filename) ? overrides.get(filename) : readFileSync(source);
|
|
} catch {
|
|
return { ok: false, async arrayBuffer() { return new ArrayBuffer(0); } };
|
|
}
|
|
return {
|
|
ok: true,
|
|
async arrayBuffer() {
|
|
return Uint8Array.from(bytes).buffer;
|
|
},
|
|
};
|
|
};
|
|
}
|
|
|
|
function filesRecursively(directory) {
|
|
return readdirSync(directory).flatMap((name) => {
|
|
const target = path.join(directory, name);
|
|
return statSync(target).isDirectory() ? filesRecursively(target) : [target];
|
|
});
|
|
}
|
|
|
|
function viewSource(html, route) {
|
|
const marker = `<section class="view" id="view-${route}"`;
|
|
const start = html.indexOf(marker);
|
|
if (start < 0) return "";
|
|
const remaining = html.slice(start + marker.length);
|
|
const nextView = remaining.indexOf('<section class="view" id="view-');
|
|
const mainEnd = remaining.indexOf("</main>");
|
|
const candidates = [nextView, mainEnd].filter((index) => index >= 0);
|
|
const end = candidates.length ? Math.min(...candidates) : remaining.length;
|
|
return html.slice(start, start + marker.length + end);
|
|
}
|
|
|
|
test("static shell exposes four semantic navigable views", () => {
|
|
const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8");
|
|
assert.match(html, /<header\b/);
|
|
assert.match(html, /<nav\b[^>]*aria-label="Dashboard views"/);
|
|
assert.match(html, /<main\b/);
|
|
assert.match(html, /<footer\b/);
|
|
for (const route of ["overview", "cohorts", "model", "methods"]) {
|
|
assert.match(html, new RegExp(`data-route="${route}"`));
|
|
assert.match(html, new RegExp(`data-view="${route}"`));
|
|
}
|
|
assert.match(html, /2025 development-sample holdout was evaluated once/i);
|
|
assert.match(html, /<strong>One-time holdout<\/strong><small>Already evaluated<\/small>/);
|
|
assert.doesNotMatch(html, /<strong>Locked test<\/strong>|<small>Final evaluation<\/small>/);
|
|
});
|
|
|
|
test("narrow layouts constrain body content while preserving local nav scrolling", () => {
|
|
const css = readFileSync(path.join(DASHBOARD_ROOT, "styles.css"), "utf8");
|
|
assert.match(css, /html\s*{[^}]*max-width:\s*100%;[^}]*overflow-x:\s*clip;/s);
|
|
assert.match(css, /body\s*{[^}]*max-width:\s*100%;[^}]*overflow-x:\s*clip;/s);
|
|
assert.match(css, /\.primary-nav__inner\s*{[^}]*overflow-x:\s*auto;/s);
|
|
assert.match(css, /\.explorer-results\s*{[^}]*min-width:\s*0;/s);
|
|
const mobile = css.match(/@media \(max-width: 660px\)\s*{[\s\S]*?(?=@media|$)/)?.[0];
|
|
assert.ok(mobile, "mobile breakpoint is present");
|
|
assert.match(mobile, /\.page-shell\s*{[^}]*width:\s*auto;[^}]*max-width:\s*calc\(100% - 2rem\);/s);
|
|
assert.match(mobile, /\.preview-banner\s*{[^}]*justify-content:\s*flex-start;/s);
|
|
assert.match(mobile, /\.chart--scorecard\s*{[^}]*overflow-x:\s*auto;/s);
|
|
assert.match(mobile, /\.chart--scorecard svg\s*{[^}]*min-width:\s*720px;/s);
|
|
assert.match(mobile, /overflow-wrap:\s*anywhere;/);
|
|
});
|
|
|
|
test("county coverage renders Utah's northeast step and geographically aligned feeds", () => {
|
|
const attributes = new Map();
|
|
const container = {
|
|
innerHTML: "",
|
|
setAttribute(name, value) {
|
|
attributes.set(name, value);
|
|
},
|
|
};
|
|
|
|
renderUtahCoverage(container, ["weber", "salt_lake", "utah"]);
|
|
|
|
assert.match(
|
|
container.innerHTML,
|
|
/data-map-feature="utah-outline" d="M44 20H153V67H226V255H44Z"/,
|
|
);
|
|
assert.doesNotMatch(container.innerHTML, /stroke-dasharray/);
|
|
assert.equal(container.innerHTML.match(/<circle\b/g)?.length, 3);
|
|
assert.match(container.innerHTML, /cx="123" cy="54"[^>]*><title>Weber County/);
|
|
assert.match(container.innerHTML, /cx="121" cy="83"[^>]*><title>Salt Lake County/);
|
|
assert.match(container.innerHTML, /cx="131" cy="108"[^>]*><title>Utah County/);
|
|
assert.match(container.innerHTML, />UTAH<\/text>/);
|
|
assert.match(attributes.get("aria-label"), /^Outline of Utah showing/);
|
|
assert.doesNotMatch(attributes.get("aria-label"), /_/);
|
|
});
|
|
|
|
test("model view declares the final model, benchmark role, and publication boundary", () => {
|
|
const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8");
|
|
const modelView = viewSource(html, "model");
|
|
assert.ok(modelView, "model section is present");
|
|
assert.match(modelView, /Calibrated logistic regression/);
|
|
assert.match(modelView, /Final prototype/);
|
|
assert.match(modelView, /Calibrated boosted tree/);
|
|
assert.match(modelView, /Benchmark only/);
|
|
assert.match(modelView, /partition used to fit Platt calibration/i);
|
|
assert.match(modelView, /not independent final-performance estimates/i);
|
|
assert.match(modelView, /No vehicle-level prediction service/);
|
|
assert.doesNotMatch(modelView, /<(?:input|select|textarea|form)\b/i);
|
|
assert.doesNotMatch(modelView, /prediction_lookup|planned tool|future approved output/i);
|
|
});
|
|
|
|
test("every result-facing view visibly labels the sample and non-population scope", () => {
|
|
const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8");
|
|
assert.match(html, /Private 10,000-vehicle development sample/);
|
|
for (const route of ["overview", "cohorts", "model", "methods"]) {
|
|
const view = viewSource(html, route);
|
|
assert.ok(view, `${route} view is present`);
|
|
assert.match(view, /scope-strip/);
|
|
assert.match(view, /sample/i);
|
|
assert.match(view, /not[^.]{0,80}population|population estimate/i);
|
|
}
|
|
assert.match(html, /not a county outcome ranking/i);
|
|
assert.match(html, /not a reliability ranking or population comparison/i);
|
|
});
|
|
|
|
test("cohort view discloses which rows are combined families", () => {
|
|
const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8");
|
|
const cohortView = viewSource(html, "cohorts");
|
|
assert.match(cohortView, /Combined model families:/);
|
|
assert.match(cohortView, /Silverado 1500; Dodge Ram 1500; Hyundai Elantra; Nissan Altima; Subaru Outback;/);
|
|
assert.match(cohortView, /Toyota 4Runner, Corolla, and Tacoma/);
|
|
assert.match(cohortView, /Exact source-label cohorts:/);
|
|
assert.match(cohortView, /Ford F150; Honda Accord and Civic; and Toyota Camry/);
|
|
});
|
|
|
|
test("all required generated assets satisfy the browser contract", () => {
|
|
const validated = validateAssetSet(assetSet());
|
|
assert.equal(validated.manifest.schema_version, SCHEMA_VERSION);
|
|
assert.equal(validated.manifest.development_preview, true);
|
|
assert.equal(validated.manifest.population_estimate_allowed, false);
|
|
assert.equal(validated.manifest.definitions.locked_test_metrics_published, false);
|
|
assert.match(validated.manifest.release_id, /^[0-9a-f]{64}$/);
|
|
assert.ok(validated.manifest.model_versions.length > 0);
|
|
assert.ok(validated.diagnostics.rows.some(
|
|
(row) => row.model === "logistic_platt" && row.partition === "calibrate",
|
|
));
|
|
assert.ok(validated.diagnostics.rows.some(
|
|
(row) => row.model === "hist_gradient_boosting_platt" && row.partition === "calibrate",
|
|
));
|
|
assert.deepEqual(
|
|
validated.filters.partitions,
|
|
["train", "tune", "calibrate"],
|
|
);
|
|
assert.ok(validated.overview.rows.length > 0);
|
|
assert.equal(validated.scorecards.rows.length, 12);
|
|
assert.deepEqual(
|
|
validated.filters.prior_make_models,
|
|
validated.scorecards.rows.map(({ prior_make, prior_model }) => ({
|
|
prior_make,
|
|
prior_model,
|
|
})),
|
|
);
|
|
});
|
|
|
|
test("browser loader verifies every raw asset digest before rendering", async () => {
|
|
const validated = await loadDashboardData({
|
|
basePath: "http://dashboard.test/public/data/",
|
|
fetchImplementation: publicDataFetch(),
|
|
cryptoImplementation: webcrypto,
|
|
});
|
|
assert.equal(validated.manifest.schema_version, SCHEMA_VERSION);
|
|
|
|
const changedOverview = Buffer.concat([
|
|
readFileSync(path.join(PUBLIC_DATA, REQUIRED_ASSETS.overview)),
|
|
Buffer.from("\n"),
|
|
]);
|
|
await assert.rejects(
|
|
loadDashboardData({
|
|
basePath: "http://dashboard.test/public/data/",
|
|
fetchImplementation: publicDataFetch(
|
|
new Map([[REQUIRED_ASSETS.overview, changedOverview]]),
|
|
),
|
|
cryptoImplementation: webcrypto,
|
|
}),
|
|
(error) =>
|
|
error instanceof DataContractError &&
|
|
/Integrity verification failed for overview_period_county\.json/.test(error.message),
|
|
);
|
|
});
|
|
|
|
test("sha256 manifest covers and matches every approved data asset", () => {
|
|
const manifest = json(REQUIRED_ASSETS.shaManifest);
|
|
const expectedNames = Object.entries(REQUIRED_ASSETS)
|
|
.filter(([name]) => name !== "shaManifest")
|
|
.map(([, filename]) => filename)
|
|
.sort();
|
|
assert.deepEqual(
|
|
manifest.files.map((file) => file.name).sort(),
|
|
expectedNames,
|
|
);
|
|
for (const entry of manifest.files) {
|
|
const digest = createHash("sha256")
|
|
.update(readFileSync(path.join(PUBLIC_DATA, entry.name)))
|
|
.digest("hex");
|
|
assert.equal(digest, entry.sha256, `${entry.name} digest`);
|
|
}
|
|
});
|
|
|
|
test("contract fails closed for missing, inconsistent, or sensitive data", () => {
|
|
const missing = assetSet();
|
|
delete missing.overview;
|
|
assert.throws(() => validateAssetSet(missing), DataContractError);
|
|
|
|
const inconsistent = assetSet();
|
|
inconsistent.ageRisk.population_estimate_allowed = true;
|
|
assert.throws(() => validateAssetSet(inconsistent), DataContractError);
|
|
|
|
const populationClaim = assetSet();
|
|
populationClaim.manifest.population_estimate_allowed = true;
|
|
assert.throws(() => validateAssetSet(populationClaim), DataContractError);
|
|
|
|
const broadenedTarget = assetSet();
|
|
broadenedTarget.manifest.definitions.target = "any inspection outcome";
|
|
assert.throws(() => validateAssetSet(broadenedTarget), DataContractError);
|
|
|
|
const sensitive = assetSet();
|
|
sensitive.scorecards.rows[0].vehicle_token = "not-public";
|
|
assert.throws(() => validateAssetSet(sensitive), DataContractError);
|
|
|
|
for (const deniedField of [
|
|
"vin",
|
|
"plate",
|
|
"zip",
|
|
"station",
|
|
"technician_id",
|
|
"inspector_id",
|
|
"row_prediction",
|
|
"probability",
|
|
"raw_json",
|
|
"credential",
|
|
"password",
|
|
"secret",
|
|
]) {
|
|
const denied = assetSet();
|
|
denied.overview.rows[0][deniedField] = "not-public";
|
|
assert.throws(() => validateAssetSet(denied), DataContractError, deniedField);
|
|
}
|
|
|
|
const extraRowField = assetSet();
|
|
extraRowField.overview.rows[0].note = "unapproved";
|
|
assert.throws(() => validateAssetSet(extraRowField), DataContractError);
|
|
|
|
const lockedMetrics = assetSet();
|
|
lockedMetrics.manifest.definitions.locked_test_metrics_published = true;
|
|
assert.throws(() => validateAssetSet(lockedMetrics), DataContractError);
|
|
|
|
const extraPartition = assetSet();
|
|
extraPartition.filters.partitions.push("locked_test");
|
|
assert.throws(() => validateAssetSet(extraPartition), DataContractError);
|
|
|
|
const missingFinalDiagnostic = assetSet();
|
|
missingFinalDiagnostic.diagnostics.rows = missingFinalDiagnostic.diagnostics.rows.filter(
|
|
(row) => !(row.model === "logistic_platt" && row.partition === "calibrate"),
|
|
);
|
|
assert.throws(() => validateAssetSet(missingFinalDiagnostic), DataContractError);
|
|
|
|
const duplicatedBenchmarkDiagnostic = assetSet();
|
|
duplicatedBenchmarkDiagnostic.diagnostics.rows.push({
|
|
...duplicatedBenchmarkDiagnostic.diagnostics.rows.find(
|
|
(row) => row.model === "hist_gradient_boosting_platt" && row.partition === "calibrate",
|
|
),
|
|
});
|
|
assert.throws(() => validateAssetSet(duplicatedBenchmarkDiagnostic), DataContractError);
|
|
|
|
const unsortedVersions = assetSet();
|
|
unsortedVersions.manifest.model_versions = ["z_v1", "a_v1"];
|
|
assert.throws(() => validateAssetSet(unsortedVersions), DataContractError);
|
|
|
|
const belowSuppression = assetSet();
|
|
belowSuppression.overview.rows[0].support_rounded =
|
|
belowSuppression.manifest.definitions.suppression_min_support - 1;
|
|
assert.throws(() => validateAssetSet(belowSuppression), DataContractError);
|
|
|
|
const incorrectlyRounded = assetSet();
|
|
incorrectlyRounded.coverage.rows[0].support_rounded += 1;
|
|
assert.throws(() => validateAssetSet(incorrectlyRounded), DataContractError);
|
|
});
|
|
|
|
test("dashboard source has no external runtime dependency or credential marker", () => {
|
|
const sourceFiles = filesRecursively(DASHBOARD_ROOT).filter(
|
|
(filename) =>
|
|
!filename.includes(`${path.sep}public${path.sep}data${path.sep}`) &&
|
|
!filename.includes(`${path.sep}tests${path.sep}`) &&
|
|
/\.(?:html|css|js|mjs)$/.test(filename),
|
|
);
|
|
for (const filename of sourceFiles) {
|
|
const source = readFileSync(filename, "utf8");
|
|
assert.doesNotMatch(source, /(?:postgres(?:ql)?:\/\/|PGPASSWORD|PGHOST|countydata\.)/i);
|
|
assert.doesNotMatch(source, /<script[^>]+src=["']https?:|@import\s+url\(["']?https?:/i);
|
|
}
|
|
});
|
|
|
|
test("public data directory contains only the approved suppressed aggregate bundle", () => {
|
|
const expected = Object.values(REQUIRED_ASSETS).sort();
|
|
const observed = readdirSync(PUBLIC_DATA).sort();
|
|
assert.deepEqual(observed, expected);
|
|
|
|
const serialized = observed
|
|
.map((filename) => readFileSync(path.join(PUBLIC_DATA, filename), "utf8"))
|
|
.join("\n");
|
|
assert.doesNotMatch(serialized, /\b[A-HJ-NPR-Z0-9]{17}\b/);
|
|
assert.doesNotMatch(serialized, /(?:postgres(?:ql)?:\/\/|PGPASSWORD|BEGIN [A-Z ]*PRIVATE KEY)/i);
|
|
});
|