diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1b6ba33 --- /dev/null +++ b/.env.example @@ -0,0 +1,14 @@ +# PostgreSQL connection values. Copy names only; never commit real credentials. +PGHOST= +PGPORT=5432 +PGDATABASE=countydata +PGUSER= +PGPASSWORD= +PGSSLMODE=require + +# Project-specific HMAC secret for replacing VINs in private analytical data. +# Encode exactly 32 random bytes as 64 hexadecimal characters. Alternatively, +# leave VIN_HASH_KEY empty and store the hex key in .secrets/vin_hmac.key (0600). +VIN_HASH_KEY= +VIN_HASH_KEY_FILE=.secrets/vin_hmac.key +VIN_HASH_KEY_VERSION=v1 diff --git a/.gitignore b/.gitignore index 2f6de41..5fc7f13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ .env +.env.* +!.env.example .venv/ __pycache__/ *.pyc .ipynb_checkpoints/ .DS_Store +/data/ +/artifacts/ +/models/ +/.secrets/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..bc9a081 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "ms-ossdata.vscode-pgsql" + ] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..f0d73de --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,113 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Utah Vehicle Health: create private VIN HMAC key", + "type": "shell", + "command": ".venv/bin/python scripts/create_vin_hash_key.py", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Countydata: test read-only connection", + "type": "shell", + "command": "set -a; source .env; set +a; .venv/bin/python scripts/check_connection.py", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: run Python tests", + "type": "shell", + "command": "PYTHONWARNINGS=error .venv/bin/python -m unittest discover -s tests -v", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: run aggregate feasibility check", + "type": "shell", + "command": "set -a; source .env; set +a; .venv/bin/python scripts/run_feasibility_check.py", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: audit source outcome mappings", + "type": "shell", + "command": "set -a; source .env; set +a; .venv/bin/python scripts/run_outcome_mapping_audit.py", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: export 10k development histories", + "type": "shell", + "command": "set -a; source .env; set +a; .venv/bin/python scripts/export_history_sample.py --vehicles 10000 --sample-percent 0.5 --overwrite", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: build private feature mart", + "type": "shell", + "command": ".venv/bin/python scripts/build_feature_mart.py --input data/private/development/history_sample_10000.csv.gz --memory-limit 4GB --threads 4 --overwrite", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: train baselines (2025 locked)", + "type": "shell", + "command": ".venv/bin/python scripts/train_baselines.py --mart data/private/marts/inspection_feature_mart.parquet --overwrite", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: train nonlinear model (2025 locked)", + "type": "shell", + "command": ".venv/bin/python scripts/train_tree_model.py --mart data/private/marts/inspection_feature_mart.parquet --overwrite", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: export sanitized dashboard preview", + "type": "shell", + "command": ".venv/bin/python scripts/export_dashboard_data.py --mart data/private/marts/inspection_feature_mart.parquet --model-manifest artifacts/private/baselines/baseline_v1/manifest.json --model-metrics artifacts/private/baselines/baseline_v1/metrics.json --model-manifest artifacts/private/tree/hist_gradient_boosting_v1/manifest.json --model-metrics artifacts/private/tree/hist_gradient_boosting_v1/metrics.json --development-preview --overwrite", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: run dashboard contract tests", + "type": "shell", + "command": "node --test dashboard/tests/contract.test.mjs", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, + { + "label": "Utah Vehicle Health: run all tests", + "dependsOrder": "sequence", + "dependsOn": [ + "Utah Vehicle Health: run Python tests", + "Utah Vehicle Health: run dashboard contract tests" + ], + "problemMatcher": [] + } + ] +} diff --git a/README.md b/README.md index 90f9348..689d8df 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,127 @@ # SummerProject2026 -Kevin Bell's summer 2026 project \ No newline at end of file +Kevin Bell's summer 2026 data project using Utah county vehicle-registration +and emissions-inspection data. + +See the [data inventory](docs/data_inventory.md) and the data-backed +[project shortlist](docs/project_options.md). + +## Selected project + +The selected direction is **Utah Vehicle Health**, an explainable model and +dashboard for the first-attempt outcome of a vehicle's next inspection episode. +Start with the [project charter](docs/project_charter.md), then use the +[modeling protocol](docs/modeling_protocol.md) and +[dashboard specification](docs/dashboard_spec.md) as the project contracts. +The latest private-sample pipeline findings are summarized in +[development results](docs/development_results.md), with intended use and +limitations consolidated in the [model card](docs/model_card.md). + +## Safe database access in VS Code + +The local `.env` file contains the standard PostgreSQL connection variables and +is intentionally excluded from Git. Never put those values in source code, +screenshots, browser JavaScript, or a Bolt project. + +1. Install the workspace-recommended **PostgreSQL** extension by Microsoft + (`ms-ossdata.vscode-pgsql`). +2. In VS Code, run **Tasks: Run Task** and choose + **Countydata: test read-only connection**. It loads `.env`, requires TLS, and + forces the PostgreSQL session into read-only mode. +3. To browse the server visually, open the PostgreSQL sidebar and add a + connection using the values in `.env`. Set SSL mode to **Require**, save the + profile at **User** scope, and store the password in macOS Keychain rather + than workspace settings. +4. Connect to `countydata` and run [sql/00_read_only_connection_check.sql](sql/00_read_only_connection_check.sql). + Safe aggregate examples are in [sql/01_safe_data_overview.sql](sql/01_safe_data_overview.sql). + +The selected project's first aggregate cohort check is +[sql/10_episode_cohort_feasibility.sql](sql/10_episode_cohort_feasibility.sql). +Private bounded extraction is handled by +[scripts/export_inspection_batch.py](scripts/export_inspection_batch.py); it +requires the project-specific `VIN_HASH_KEY` described in `.env.example` and +never writes a raw VIN. + +The current database login has write privileges even though this project only +needs reads. Ask the database administrator for a dedicated read-only role +before connecting any deployed service. Until then, always use explicit +read-only transactions. + +## Reproducible development workflow + +Create the local environment once: + +```bash +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +.venv/bin/python scripts/create_vin_hash_key.py +``` + +The recommended way to run the pipeline is **Tasks: Run Task** in VS Code. The +tasks preserve the intended order: + +1. Test the read-only TLS connection. +2. Run the source outcome-mapping and aggregate feasibility audits. +3. Export the private 10,000-vehicle development histories. +4. Build the private DuckDB warehouse and Parquet feature mart. +5. Run every test. +6. Train the baselines while leaving the 2025 test partition locked. +7. Train the nonlinear comparison while leaving the same partition locked. + +The development history sample is deliberately not population-representative. +It exists to exercise feature engineering and modeling before a complete, +contiguous bounded extraction is approved. Generated extracts, marts, database +files, keys, and model artifacts remain under Git-ignored private paths. + +The 2025 metrics are not calculated by the normal training task. Unlocking them +requires the conspicuous `--evaluate-locked` flag after the feature set, +hyperparameters, calibration method, and reporting plan are frozen. +That gate was opened once on the page-sampled development extract during live +verification; the normal artifact paths were then regenerated closed. The +audit trail is in [development results](docs/development_results.md), and no +further 2025-informed tuning is permitted. + +## Sanitized dashboard preview + +After training both closed model bundles, run **Tasks: Run Task** and choose +**Utah Vehicle Health: export sanitized dashboard preview**. The exporter +publishes only suppression-reviewed JSON under `dashboard/public/data`; it +refuses the sampled mart unless the explicit development-preview flag is used +and refuses model artifacts containing holdout metrics. + +Preview the site without exposing the repository root: + +```bash +node dashboard/server.mjs +``` + +Then open `http://localhost:4173`. Do not serve the repository root, because it +contains the local connection profile and private ignored directories. + +The checked-in bundle is visibly labeled as a non-population development +preview. Before a real public-data release, rerun the frozen pipeline on an +approved complete extraction and pass the publication review documented in +[the dashboard specification](docs/dashboard_spec.md). + +Local and Bolt handoff instructions are in +[dashboard/README.md](dashboard/README.md). Bolt can import a GitHub repository, +but the hosted site root must be the `dashboard` directory; never copy `.env`, +private data, model artifacts, or database tooling into a web project. + +## Data safety and publishing + +The source contains direct identifiers and operational data, including VINs, +plates, user records, sessions, and upload metadata. A public dashboard should +contain only de-identified aggregates and model outputs with minimum group-size +suppression. + +Recommended deployment flow: + +```text +countydata (read-only) -> local ETL/modeling -> sanitized aggregate tables/files + -> Bolt-hosted dashboard +``` + +Do not connect a browser directly to `countydata`. If the dashboard must refresh +automatically, use a scheduled backend job with a dedicated read-only database +role and copy only approved aggregate results into the hosted application. diff --git a/dashboard/README.md b/dashboard/README.md new file mode 100644 index 0000000..95150b3 --- /dev/null +++ b/dashboard/README.md @@ -0,0 +1,67 @@ +# Utah Vehicle Health dashboard + +This directory is a dependency-free static prototype. It uses semantic HTML, +CSS, vanilla ES modules, and inline SVG generated from approved aggregate JSON. +It has no database client, server credential, analytics SDK, or external CDN. + +## Run locally from the repository root + +```bash +node dashboard/server.mjs +``` + +Open `http://127.0.0.1:4173`. Do not open `index.html` directly with a `file:` +URL; browsers will block the JSON module requests. To use another port: + +```bash +PORT=8080 node dashboard/server.mjs +``` + +Run the dependency-free contract checks with: + +```bash +node --test dashboard/tests/contract.test.mjs +``` + +## Public-data boundary + +The browser requires and validates these files beneath `public/data/`: + +- `data_manifest.json` +- `overview_period_county.json` +- `age_risk_curve.json` +- `cohort_scorecard.json` +- `model_diagnostics.json` +- `coverage_quality.json` +- `filter_catalog.json` +- `sha256_manifest.json` + +If any file is missing, malformed, has inconsistent publication flags, or +contains a denied identifier-like field, the dashboard shows an unavailable +state and no estimates. Regenerate assets with the repository's private local +pipeline; never hand-edit public JSON to bypass suppression. + +The next-test estimator is intentionally disabled. It must remain disabled +until a separately reviewed and suppressed `prediction_lookup` contract is +approved. Do not add a connection from this site to `countydata`. + +## Import into Bolt + +The recommended publication boundary is a separate static Bolt project (and, +ideally, a separate deployment repository) containing only the contents of +`dashboard/`. Upload or copy this directory's contents so `index.html` is at the +new project root. There is no install or build step. If Bolt asks for a preview +command, use `node server.mjs` with `HOST=0.0.0.0`; Bolt supplies `PORT`. + +Before publishing, confirm that `/public/data/data_manifest.json` resolves, +the development-preview banner remains visible, and the contract test passes. +The generated JSON bundle must also complete its privacy review. + +Importing the whole source repository is a discouraged fallback because Bolt +would receive SQL, database tooling, and other files that are not needed by the +public site. If that has already happened, set the project/working root to +`dashboard`, never serve the repository root, and create a dashboard-only +project before production publication. Do not copy private `data/`, +`artifacts/`, `models/`, `.env`, or database tooling into the public project. + +No publishing action is performed by this repository. diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000..c205cea --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,379 @@ + + + + + + + + Overview · Utah Vehicle Health + + + + + + + + + + +
+
+
+
+
+

Inspection outcomes over time

+

A clearer view of the next inspection

+

Explore aggregate first-attempt outcomes for returning vehicles in participating Utah county feeds. Non-pass combines fail, reject, and abort.

+
+
+ Data through + Unavailable + Waiting for manifest +
+
+ + + +
+
+ Published support + + Rounded total; suppressed cells omitted +
+
+ Pass rate + + Recognized first-attempt outcomes +
+
+ Non-pass rate + + Fail + reject + abort +
+
+ Covered counties + + Not statewide coverage +
+
+ +
+
+
+
+

Quarterly outcomes

+

First-attempt non-pass trend

+
+
+ Non-pass (fail + reject + abort) +
+
+ +

Partial periods and source changes are called out when present.

+
+ +
+
+
+

Feed availability

+

County coverage

+
+
+ +
+

Teal counties appear in the approved inspection feeds. Gray counties are unavailable, not zero.

+
+ +
+
+
+

Age pattern

+

Non-pass risk by vehicle age

+
+
+ +

The development bundle does not include confidence intervals; published aggregate rates are shown as points.

+
+
+ + +
+
+ + + + + + +
+ + + +
+ + + diff --git a/dashboard/js/app.js b/dashboard/js/app.js new file mode 100644 index 0000000..5e120df --- /dev/null +++ b/dashboard/js/app.js @@ -0,0 +1,352 @@ +import { loadDashboardData } from "./data.js"; +import { + aggregateOverview, + compactNumber, + percent, + renderAgeRisk, + renderCohortDotPlot, + renderCoverageHeatmap, + renderNoCalibration, + renderOutcomeTrend, + renderUtahCoverage, +} from "./charts.js"; + +const ROUTES = Object.freeze({ + overview: "Overview", + reliability: "Reliability explorer", + estimator: "Next-test estimator", + methods: "Data & methods", +}); + +const UTAH_COUNTIES = [ + "Beaver", + "Box Elder", + "Cache", + "Carbon", + "Daggett", + "Davis", + "Duchesne", + "Emery", + "Garfield", + "Grand", + "Iron", + "Juab", + "Kane", + "Millard", + "Morgan", + "Piute", + "Rich", + "Salt Lake", + "San Juan", + "Sanpete", + "Sevier", + "Summit", + "Tooele", + "Uintah", + "Utah", + "Wasatch", + "Washington", + "Wayne", + "Weber", +]; + +let dashboardData = null; +let visibleScorecards = []; + +function byId(id) { + return document.getElementById(id); +} + +function setText(id, value) { + const element = byId(id); + if (element) element.textContent = value; +} + +function normalizeCounty(value) { + return String(value).trim().toLowerCase().replaceAll("_", " ").replace(/\s+/g, " "); +} + +function displayCategory(value) { + return String(value) + .replaceAll("_", " ") + .toLowerCase() + .replace(/\b\w/g, (letter) => letter.toUpperCase()); +} + +function routeFromHash() { + const candidate = window.location.hash.replace(/^#/, "").toLowerCase(); + return Object.hasOwn(ROUTES, candidate) ? candidate : "overview"; +} + +function showRoute({ announce = true, focus = false } = {}) { + const route = routeFromHash(); + for (const view of document.querySelectorAll("[data-view]")) { + view.hidden = view.dataset.view !== route; + } + for (const link of document.querySelectorAll("[data-route]")) { + if (link.dataset.route === route) link.setAttribute("aria-current", "page"); + else link.removeAttribute("aria-current"); + } + document.title = `${ROUTES[route]} · Utah Vehicle Health`; + if (announce) setText("route-announcer", `${ROUTES[route]} view`); + if (focus) { + const heading = document.querySelector(`[data-view="${route}"] h1`); + if (heading) { + heading.setAttribute("tabindex", "-1"); + heading.focus({ preventScroll: true }); + heading.scrollIntoView({ behavior: "smooth", block: "start" }); + } + } +} + +function initializeRouting() { + showRoute({ announce: false, focus: false }); + window.addEventListener("hashchange", () => showRoute({ focus: true })); +} + +function setHeaderStatus(state, message) { + const status = byId("header-status"); + if (!status) return; + status.dataset.state = state; + const text = status.querySelector("span:last-child"); + if (text) text.textContent = message; +} + +function setPreviewBanner(manifest) { + const preview = manifest.development_preview || !manifest.population_estimate_allowed; + const banner = byId("preview-banner"); + banner.hidden = !preview; + if (!preview) return; + setText("preview-title", "Development preview"); + setText( + "preview-copy", + "These suppressed sample aggregates are not population estimates and must not be used for individual decisions.", + ); +} + +function setUnavailable(error) { + const safeMessage = + error instanceof Error && error.message + ? `${error.message} No estimates are being shown.` + : "The approved aggregate files could not be validated. No estimates are being shown."; + for (const state of document.querySelectorAll("[data-unavailable]")) { + state.hidden = false; + const copy = state.querySelector("[data-unavailable-message]"); + if (copy) copy.textContent = safeMessage; + } + for (const region of document.querySelectorAll("[data-requires-data]")) { + region.hidden = true; + } + const banner = byId("preview-banner"); + banner.hidden = false; + setText("preview-title", "Data unavailable"); + setText("preview-copy", "This static shell could not validate every required public aggregate. No fallback or estimated values are displayed."); + setHeaderStatus("error", "Approved data unavailable"); +} + +function populateSelect(id, values, formatter = displayCategory) { + const select = byId(id); + if (!select) return; + for (const value of values) { + const option = document.createElement("option"); + option.value = String(value); + option.textContent = formatter(value); + select.append(option); + } +} + +function disableUnsupportedFilter(id, explanation) { + const select = byId(id); + if (!select) return; + select.disabled = true; + select.title = explanation; + select.options[0].textContent = explanation; +} + +function renderCountyList(coveredCounties) { + const container = byId("county-coverage-list"); + container.replaceChildren(); + const normalized = new Set(coveredCounties.map(normalizeCounty)); + for (const county of UTAH_COUNTIES) { + const chip = document.createElement("span"); + chip.className = "county-chip"; + chip.dataset.covered = String(normalized.has(normalizeCounty(county))); + chip.textContent = county; + container.append(chip); + } +} + +function renderOverview(data) { + const rows = data.overview.rows; + const periods = aggregateOverview(rows); + const totalSupport = rows.reduce((sum, row) => sum + row.support_rounded, 0); + const weightedNonpass = rows.reduce( + (sum, row) => sum + row.support_rounded * row.nonpass_rate, + 0, + ); + const nonpassRate = totalSupport > 0 ? weightedNonpass / totalSupport : 0; + const counties = [...new Set(rows.map((row) => row.public_county))].sort(); + const latest = periods[periods.length - 1]; + + setText("kpi-eligible", `≈${compactNumber(totalSupport)}`); + setText("kpi-pass", `≈${percent(1 - nonpassRate)}`); + setText("kpi-nonpass", `≈${percent(nonpassRate)}`); + setText("kpi-counties", String(counties.length)); + setText("data-cutoff", latest ? `${latest.year} Q${latest.quarter}` : "Unavailable"); + setText( + "model-version", + `Release ${data.manifest.release_id.slice(0, 8)} · ${data.manifest.model_versions.join(" / ")}`, + ); + renderOutcomeTrend(byId("outcome-trend-chart"), rows); + setText( + "outcome-trend-note", + "The current approved bundle publishes binary non-pass rates only. Four-class outcome mix and blank rates are not inferred or displayed.", + ); + renderAgeRisk(byId("age-risk-chart"), data.ageRisk.rows); + renderUtahCoverage(byId("utah-coverage-map"), counties); + renderCountyList(counties); +} + +function scorecardLabel(row) { + return `${row.prior_make} ${row.prior_model}`.trim(); +} + +function sortedScorecards(rows, mode) { + const sorted = [...rows]; + if (mode === "risk-desc") sorted.sort((a, b) => b.nonpass_rate - a.nonpass_rate); + else if (mode === "risk-asc") sorted.sort((a, b) => a.nonpass_rate - b.nonpass_rate); + else if (mode === "name") sorted.sort((a, b) => scorecardLabel(a).localeCompare(scorecardLabel(b))); + else sorted.sort((a, b) => b.support_rounded - a.support_rounded); + return sorted; +} + +function renderScorecardCards(rows) { + const container = byId("cohort-cards"); + container.replaceChildren(); + for (const row of rows.slice(0, 18)) { + const article = document.createElement("article"); + article.className = "cohort-card"; + const heading = document.createElement("h3"); + heading.textContent = scorecardLabel(row); + const meta = document.createElement("div"); + meta.className = "cohort-card__meta"; + meta.textContent = `${compactNumber(row.support_rounded)} rounded support`; + const bar = document.createElement("div"); + bar.className = "risk-bar"; + bar.setAttribute("aria-hidden", "true"); + const fill = document.createElement("span"); + fill.style.width = `${Math.min(100, row.nonpass_rate * 100)}%`; + bar.append(fill); + const value = document.createElement("div"); + value.className = "cohort-card__value"; + const strong = document.createElement("strong"); + strong.textContent = percent(row.nonpass_rate); + const small = document.createElement("small"); + small.textContent = "Observed non-pass"; + value.append(strong, small); + article.append(heading, meta, bar, value); + container.append(article); + } +} + +function updateExplorer() { + if (!dashboardData) return; + const query = byId("cohort-search").value.trim().toLowerCase(); + const sortMode = byId("result-sort").value; + const matches = dashboardData.scorecards.rows.filter((row) => + scorecardLabel(row).toLowerCase().includes(query), + ); + visibleScorecards = sortedScorecards(matches, sortMode); + setText( + "result-summary", + `${visibleScorecards.length} supported cohort${visibleScorecards.length === 1 ? "" : "s"}; showing up to 18 cards and 12 chart rows.`, + ); + byId("empty-results").hidden = visibleScorecards.length > 0; + renderCohortDotPlot(byId("cohort-dot-plot"), visibleScorecards); + renderScorecardCards(visibleScorecards); +} + +function initializeExplorer(data) { + populateSelect("filter-county", data.filters.public_counties); + populateSelect("filter-age", data.filters.age_bands, (value) => String(value)); + populateSelect( + "filter-period", + data.filters.periods.map((period) => period.year), + (value) => String(value), + ); + + // Current scorecards are make/model aggregates only. These planned controls + // remain visible but disabled so the UI never implies unsupported slicing. + disableUnsupportedFilter("filter-county", "Unavailable at current scorecard grain"); + disableUnsupportedFilter("filter-age", "Unavailable at current scorecard grain"); + disableUnsupportedFilter("filter-fuel", "Fuel not published in this bundle"); + disableUnsupportedFilter("filter-program", "Program not published in this bundle"); + disableUnsupportedFilter("filter-period", "Period not published in this scorecard"); + const adjusted = document.querySelector('input[name="risk-view"][value="adjusted"]'); + adjusted.disabled = true; + adjusted.parentElement.title = "Model-adjusted cohort scorecards are not published."; + + byId("cohort-search").addEventListener("input", updateExplorer); + byId("result-sort").addEventListener("change", updateExplorer); + byId("reset-filters").addEventListener("click", () => { + byId("explorer-filters").reset(); + byId("cohort-search").value = ""; + byId("result-sort").value = "support"; + updateExplorer(); + }); + updateExplorer(); +} + +function chooseDiagnostic(rows) { + const partitionRank = { calibrate: 3, tune: 2, train: 1 }; + return [...rows].sort((left, right) => { + const preferredLeft = left.model === "logistic_platt" ? 1 : 0; + const preferredRight = right.model === "logistic_platt" ? 1 : 0; + return ( + preferredRight - preferredLeft || + (partitionRank[right.partition] ?? 0) - (partitionRank[left.partition] ?? 0) + ); + })[0]; +} + +function renderMethods(data) { + renderCoverageHeatmap(byId("coverage-heatmap"), data.coverage.rows); + const diagnostic = chooseDiagnostic(data.diagnostics.rows); + setText( + "diagnostic-scope-label", + diagnostic + ? `${diagnostic.partition === "calibrate" ? "Calibration cohort" : "Development diagnostics"} · ${diagnostic.model}` + : "Development diagnostics · model unavailable", + ); + setText("diagnostic-ap", diagnostic ? diagnostic.average_precision.toFixed(3) : "—"); + setText("diagnostic-brier", diagnostic ? diagnostic.brier.toFixed(3) : "—"); + setText("diagnostic-n", "Not published"); + renderNoCalibration(byId("calibration-chart")); +} + +function renderDashboard(data) { + dashboardData = data; + setPreviewBanner(data.manifest); + renderOverview(data); + initializeExplorer(data); + renderMethods(data); + setHeaderStatus( + "ready", + data.manifest.development_preview ? "Validated development aggregates" : "Validated public aggregates", + ); +} + +async function initialize() { + initializeRouting(); + try { + const data = await loadDashboardData(); + renderDashboard(data); + } catch (error) { + console.error("Dashboard data contract rejected the public assets.", error); + setUnavailable(error); + } +} + +initialize(); + +export { displayCategory, normalizeCounty, routeFromHash }; diff --git a/dashboard/js/charts.js b/dashboard/js/charts.js new file mode 100644 index 0000000..2372c89 --- /dev/null +++ b/dashboard/js/charts.js @@ -0,0 +1,281 @@ +const SVG_NS = "http://www.w3.org/2000/svg"; + +function escapeText(value) { + return String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function percent(value, digits = 1) { + return new Intl.NumberFormat("en-US", { + style: "percent", + minimumFractionDigits: digits, + maximumFractionDigits: digits, + }).format(value); +} + +function compactNumber(value) { + return new Intl.NumberFormat("en-US", { + notation: value >= 10_000 ? "compact" : "standard", + maximumFractionDigits: 1, + }).format(value); +} + +function emptyChart(container, message) { + container.innerHTML = `

${escapeText(message)}

`; + container.setAttribute("aria-label", message); +} + +function aggregateOverview(rows) { + const groups = new Map(); + for (const row of rows) { + const key = `${row.year}-Q${row.quarter}`; + const current = groups.get(key) ?? { + label: `${row.year} Q${row.quarter}`, + year: row.year, + quarter: row.quarter, + support: 0, + weightedRisk: 0, + }; + current.support += row.support_rounded; + current.weightedRisk += row.support_rounded * row.nonpass_rate; + groups.set(key, current); + } + return [...groups.values()] + .map((group) => ({ + ...group, + risk: group.support > 0 ? group.weightedRisk / group.support : 0, + })) + .sort((left, right) => left.year - right.year || left.quarter - right.quarter); +} + +export function renderOutcomeTrend(container, rows) { + const points = aggregateOverview(rows); + if (points.length < 2) { + emptyChart(container, "Not enough approved periods to draw a trend."); + return; + } + const width = 920; + const height = 330; + const margin = { top: 24, right: 22, bottom: 54, left: 58 }; + const plotWidth = width - margin.left - margin.right; + const plotHeight = height - margin.top - margin.bottom; + const maxRisk = Math.max(0.05, ...points.map((point) => point.risk)); + const yMax = Math.min(1, Math.ceil(maxRisk * 20) / 20); + const x = (index) => margin.left + (index / (points.length - 1)) * plotWidth; + const y = (value) => margin.top + plotHeight - (value / yMax) * plotHeight; + const path = points + .map((point, index) => `${index === 0 ? "M" : "L"}${x(index).toFixed(1)},${y(point.risk).toFixed(1)}`) + .join(" "); + + const yTicks = Array.from({ length: 5 }, (_, index) => (index / 4) * yMax); + const tickEvery = Math.max(1, Math.ceil(points.length / 8)); + const labelled = points.filter( + (_point, index) => index % tickEvery === 0 || index === points.length - 1, + ); + const description = points + .map((point) => `${point.label}: ${percent(point.risk)}`) + .join("; "); + + container.innerHTML = ` + `; + container.setAttribute( + "aria-label", + `Quarterly aggregate non-pass trend. ${description}`, + ); +} + +export function renderAgeRisk(container, rows) { + if (!rows.length) { + emptyChart(container, "No supported vehicle-age bands are available."); + return; + } + const points = [...rows]; + const width = 600; + const height = 300; + const margin = { top: 24, right: 20, bottom: 58, left: 52 }; + const plotWidth = width - margin.left - margin.right; + const plotHeight = height - margin.top - margin.bottom; + const maxRisk = Math.max(0.05, ...points.map((point) => point.nonpass_rate)); + const yMax = Math.min(1, Math.ceil(maxRisk * 20) / 20); + const denominator = Math.max(1, points.length - 1); + const x = (index) => margin.left + (index / denominator) * plotWidth; + const y = (value) => margin.top + plotHeight - (value / yMax) * plotHeight; + const path = points + .map((point, index) => `${index === 0 ? "M" : "L"}${x(index)},${y(point.nonpass_rate)}`) + .join(" "); + + container.innerHTML = ` + `; + container.setAttribute( + "aria-label", + `Non-pass risk by vehicle-age band. ${points + .map((point) => `${point.age_band}: ${percent(point.nonpass_rate)}`) + .join("; ")}`, + ); +} + +export function renderCohortDotPlot(container, rows) { + const points = rows.slice(0, 12); + if (!points.length) { + emptyChart(container, "No supported cohorts match the current filters."); + return; + } + const width = 820; + const rowHeight = 34; + const margin = { top: 24, right: 60, bottom: 42, left: 220 }; + const height = margin.top + margin.bottom + points.length * rowHeight; + const plotWidth = width - margin.left - margin.right; + const maxRisk = Math.max(0.05, ...points.map((point) => point.nonpass_rate)); + const xMax = Math.min(1, Math.ceil(maxRisk * 20) / 20); + const x = (value) => margin.left + (value / xMax) * plotWidth; + + container.innerHTML = ` + `; + container.setAttribute( + "aria-label", + `Ranked supported cohort non-pass risk. ${points + .map((point) => `${point.prior_make} ${point.prior_model}: ${percent(point.nonpass_rate)}`) + .join("; ")}`, + ); +} + +const COUNTY_POINTS = { + cache: [154, 40], + weber: [137, 86], + davis: [126, 108], + salt_lake: [133, 133], + "salt lake": [133, 133], + utah: [140, 170], +}; + +export function renderUtahCoverage(container, coveredCounties) { + const normalized = new Set(coveredCounties.map((county) => county.toLowerCase())); + const points = [...normalized] + .map((county) => ({ county, coordinates: COUNTY_POINTS[county] })) + .filter((item) => item.coordinates); + container.innerHTML = ` + `; + container.setAttribute( + "aria-label", + coveredCounties.length + ? `Utah feed coverage includes ${coveredCounties.join(", ")}. Other counties are unavailable.` + : "No county feed coverage is available.", + ); +} + +export function renderCoverageHeatmap(container, rows) { + if (!rows.length) { + emptyChart(container, "No source-era coverage rows are available."); + return; + } + const years = [...new Set(rows.map((row) => row.year))].sort((a, b) => a - b); + const eras = [...new Set(rows.map((row) => row.source_era))].sort(); + const lookup = new Map(rows.map((row) => [`${row.source_era}-${row.year}`, row])); + const maxSupport = Math.max(1, ...rows.map((row) => row.support_rounded)); + const columns = `90px repeat(${years.length}, minmax(34px, 1fr))`; + container.innerHTML = ` +
+ ${years.map((year) => `${year}`).join("")} + ${eras + .map( + (era) => `${escapeText(era)}${years + .map((year) => { + const row = lookup.get(`${era}-${year}`); + if (!row) return ``; + const alpha = 0.15 + 0.75 * Math.sqrt(row.support_rounded / maxSupport); + return ``; + }) + .join("")}`, + ) + .join("")} +
`; + container.setAttribute( + "aria-label", + `Source-era coverage from ${years[0]} through ${years[years.length - 1]} for ${eras.join(", ")}.`, + ); +} + +export function renderNoCalibration(container) { + emptyChart( + container, + "Calibration-bin data is not published in this development bundle. No calibration curve is shown.", + ); +} + +export { aggregateOverview, compactNumber, percent }; diff --git a/dashboard/js/data.js b/dashboard/js/data.js new file mode 100644 index 0000000..21c2133 --- /dev/null +++ b/dashboard/js/data.js @@ -0,0 +1,574 @@ +const SCHEMA_VERSION = "dashboard_data_v1"; +const APPROVED_PARTITIONS = Object.freeze(["train", "tune", "calibrate"]); +const ENVELOPE_KEYS = Object.freeze([ + "development_preview", + "population_estimate_allowed", + "schema_version", +]); + +export const REQUIRED_ASSETS = Object.freeze({ + manifest: "data_manifest.json", + overview: "overview_period_county.json", + ageRisk: "age_risk_curve.json", + scorecards: "cohort_scorecard.json", + diagnostics: "model_diagnostics.json", + coverage: "coverage_quality.json", + filters: "filter_catalog.json", + shaManifest: "sha256_manifest.json", +}); + +const SENSITIVE_KEY_PARTS = new Set([ + "address", + "certificate", + "email", + "internal", + "ip", + "owner", + "pid", + "plate", + "raw", + "session", + "station", + "token", + "user", + "vin", + "zip", +]); + +const ASSET_VALIDATORS = { + overview: (value) => validateRows(value, validateOverviewRow, "overview_period_county"), + ageRisk: (value) => validateRows(value, validateAgeRiskRow, "age_risk_curve"), + scorecards: (value) => validateRows(value, validateScorecardRow, "cohort_scorecard"), + diagnostics: (value) => validateRows(value, validateDiagnosticRow, "model_diagnostics"), + coverage: (value) => validateRows(value, validateCoverageRow, "coverage_quality"), + filters: validateFilterCatalog, + shaManifest: validateShaManifest, +}; + +export class DataContractError extends Error { + constructor(message) { + super(message); + this.name = "DataContractError"; + } +} + +function isPlainObject(value) { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function requirePlainObject(value, label) { + if (!isPlainObject(value)) { + throw new DataContractError(`${label} must be a JSON object.`); + } +} + +function requireExactKeys(value, expectedKeys, label) { + const observed = Object.keys(value).sort(); + const expected = [...expectedKeys].sort(); + if (JSON.stringify(observed) !== JSON.stringify(expected)) { + throw new DataContractError(`${label} does not match the approved field schema.`); + } +} + +function requireExactStringSet(value, expectedValues, label) { + validateStringArray(value, label); + const observed = [...value].sort(); + const expected = [...expectedValues].sort(); + if (JSON.stringify(observed) !== JSON.stringify(expected)) { + throw new DataContractError(`${label} does not match the approved values.`); + } +} + +function requireBoolean(value, label) { + if (typeof value !== "boolean") { + throw new DataContractError(`${label} must be true or false.`); + } +} + +function requireString(value, label) { + if (typeof value !== "string" || value.trim() === "") { + throw new DataContractError(`${label} must be a non-empty string.`); + } +} + +function requireSafeModelVersion(value, label) { + requireString(value, label); + if (!/^[A-Za-z0-9._-]{1,64}$/.test(value)) { + throw new DataContractError(`${label} is not an approved model version.`); + } +} + +function requireNumber(value, label, { min = -Infinity, max = Infinity } = {}) { + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) { + throw new DataContractError(`${label} must be a finite number in the approved range.`); + } +} + +function requireInteger(value, label, { min = -Infinity, max = Infinity } = {}) { + requireNumber(value, label, { min, max }); + if (!Number.isInteger(value)) { + throw new DataContractError(`${label} must be an integer.`); + } +} + +function validateEnvelope(value, label) { + requirePlainObject(value, label); + if (value.schema_version !== SCHEMA_VERSION) { + throw new DataContractError(`${label} uses an unsupported schema version.`); + } + requireBoolean(value.development_preview, `${label}.development_preview`); + requireBoolean( + value.population_estimate_allowed, + `${label}.population_estimate_allowed`, + ); + rejectSensitiveKeys(value, label); +} + +function validateRows(value, rowValidator, label) { + validateEnvelope(value, label); + requireExactKeys(value, [...ENVELOPE_KEYS, "rows"], label); + if (!Array.isArray(value.rows)) { + throw new DataContractError(`${label}.rows must be an array.`); + } + value.rows.forEach((row, index) => { + requirePlainObject(row, `${label}.rows[${index}]`); + rowValidator(row, `${label}.rows[${index}]`); + }); + return value; +} + +function validateOverviewRow(row, label) { + requireExactKeys( + row, + ["year", "quarter", "public_county", "support_rounded", "nonpass_rate"], + label, + ); + requireInteger(row.year, `${label}.year`, { min: 2010, max: 2100 }); + requireInteger(row.quarter, `${label}.quarter`, { min: 1, max: 4 }); + requireString(row.public_county, `${label}.public_county`); + requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 }); + requireNumber(row.nonpass_rate, `${label}.nonpass_rate`, { min: 0, max: 1 }); +} + +function validateAgeRiskRow(row, label) { + requireExactKeys(row, ["age_band", "support_rounded", "nonpass_rate"], label); + requireString(row.age_band, `${label}.age_band`); + requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 }); + requireNumber(row.nonpass_rate, `${label}.nonpass_rate`, { min: 0, max: 1 }); +} + +function validateScorecardRow(row, label) { + requireExactKeys( + row, + ["prior_make", "prior_model", "support_rounded", "nonpass_rate"], + label, + ); + requireString(row.prior_make, `${label}.prior_make`); + requireString(row.prior_model, `${label}.prior_model`); + requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 }); + requireNumber(row.nonpass_rate, `${label}.nonpass_rate`, { min: 0, max: 1 }); +} + +function validateDiagnosticRow(row, label) { + requireExactKeys( + row, + [ + "model", + "partition", + "average_precision", + "brier", + "log_loss", + "roc_auc", + "top_10_capture", + ], + label, + ); + requireString(row.model, `${label}.model`); + requireString(row.partition, `${label}.partition`); + if (!APPROVED_PARTITIONS.includes(row.partition)) { + throw new DataContractError(`${label}.partition is not approved for this public schema.`); + } + for (const metric of [ + "average_precision", + "brier", + "log_loss", + "roc_auc", + "top_10_capture", + ]) { + requireNumber(row[metric], `${label}.${metric}`, { min: 0 }); + } + if (row.average_precision > 1 || row.brier > 1 || row.roc_auc > 1 || row.top_10_capture > 1) { + throw new DataContractError(`${label} contains a probability metric above 1.`); + } +} + +function validateCoverageRow(row, label) { + requireExactKeys( + row, + [ + "year", + "source_era", + "support_rounded", + "labeled_rate", + "overall_result_share", + "utah_obd_proxy_share", + ], + label, + ); + requireInteger(row.year, `${label}.year`, { min: 2010, max: 2100 }); + requireString(row.source_era, `${label}.source_era`); + requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 }); + for (const metric of ["labeled_rate", "overall_result_share", "utah_obd_proxy_share"]) { + requireNumber(row[metric], `${label}.${metric}`, { min: 0, max: 1 }); + } +} + +function validateStringArray(value, label) { + if (!Array.isArray(value)) throw new DataContractError(`${label} must be an array.`); + value.forEach((item, index) => requireString(item, `${label}[${index}]`)); +} + +function validateFilterCatalog(value) { + const label = "filter_catalog"; + validateEnvelope(value, label); + requireExactKeys( + value, + [ + ...ENVELOPE_KEYS, + "public_counties", + "age_bands", + "prior_make_models", + "periods", + "models", + "partitions", + ], + label, + ); + validateStringArray(value.public_counties, `${label}.public_counties`); + validateStringArray(value.age_bands, `${label}.age_bands`); + validateStringArray(value.models, `${label}.models`); + requireExactStringSet(value.partitions, APPROVED_PARTITIONS, `${label}.partitions`); + if (!Array.isArray(value.prior_make_models)) { + throw new DataContractError(`${label}.prior_make_models must be an array.`); + } + value.prior_make_models.forEach((row, index) => { + requirePlainObject(row, `${label}.prior_make_models[${index}]`); + requireExactKeys( + row, + ["prior_make", "prior_model"], + `${label}.prior_make_models[${index}]`, + ); + requireString(row.prior_make, `${label}.prior_make_models[${index}].prior_make`); + requireString(row.prior_model, `${label}.prior_make_models[${index}].prior_model`); + }); + if (!Array.isArray(value.periods)) { + throw new DataContractError(`${label}.periods must be an array.`); + } + value.periods.forEach((period, index) => { + requirePlainObject(period, `${label}.periods[${index}]`); + requireExactKeys(period, ["year", "quarters"], `${label}.periods[${index}]`); + requireInteger(period.year, `${label}.periods[${index}].year`, { + min: 2010, + max: 2100, + }); + if (!Array.isArray(period.quarters) || period.quarters.length === 0) { + throw new DataContractError(`${label}.periods[${index}].quarters must be non-empty.`); + } + period.quarters.forEach((quarter, quarterIndex) => + requireInteger(quarter, `${label}.periods[${index}].quarters[${quarterIndex}]`, { + min: 1, + max: 4, + }), + ); + }); + return value; +} + +function validateShaManifest(value) { + const label = "sha256_manifest"; + validateEnvelope(value, label); + requireExactKeys(value, [...ENVELOPE_KEYS, "files"], label); + if (!Array.isArray(value.files)) { + throw new DataContractError(`${label}.files must be an array.`); + } + const expectedNames = Object.entries(REQUIRED_ASSETS) + .filter(([name]) => name !== "shaManifest") + .map(([, filename]) => filename) + .sort(); + const observedNames = []; + for (const [index, file] of value.files.entries()) { + requirePlainObject(file, `${label}.files[${index}]`); + requireExactKeys(file, ["name", "sha256"], `${label}.files[${index}]`); + requireString(file.name, `${label}.files[${index}].name`); + requireString(file.sha256, `${label}.files[${index}].sha256`); + if (!/^[0-9a-f]{64}$/.test(file.sha256)) { + throw new DataContractError(`${label}.files[${index}].sha256 is invalid.`); + } + observedNames.push(file.name); + } + observedNames.sort(); + if (JSON.stringify(observedNames) !== JSON.stringify(expectedNames)) { + throw new DataContractError(`${label} does not cover the exact approved asset set.`); + } + return value; +} + +function rejectSensitiveKeys(value, label, seen = new WeakSet()) { + if (value === null || typeof value !== "object") return; + if (seen.has(value)) return; + seen.add(value); + if (Array.isArray(value)) { + value.forEach((item, index) => rejectSensitiveKeys(item, `${label}[${index}]`, seen)); + return; + } + for (const [key, child] of Object.entries(value)) { + const normalizedParts = key.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); + if (normalizedParts.some((part) => SENSITIVE_KEY_PARTS.has(part))) { + throw new DataContractError(`${label} contains a field outside the public data contract.`); + } + rejectSensitiveKeys(child, `${label}.${key}`, seen); + } +} + +export function validateAssetSet(rawAssets) { + requirePlainObject(rawAssets, "asset set"); + requireExactKeys(rawAssets, Object.keys(REQUIRED_ASSETS), "asset set"); + const manifest = rawAssets.manifest; + validateEnvelope(manifest, "data_manifest"); + requireExactKeys( + manifest, + [ + ...ENVELOPE_KEYS, + "assets", + "data_scope", + "definitions", + "model_versions", + "release_id", + ], + "data_manifest", + ); + if (Object.hasOwn(manifest, "rows")) { + throw new DataContractError("data_manifest must not contain row data."); + } + requirePlainObject(manifest.data_scope, "data_manifest.data_scope"); + requirePlainObject(manifest.definitions, "data_manifest.definitions"); + requireString(manifest.release_id, "data_manifest.release_id"); + if (!/^[0-9a-f]{64}$/.test(manifest.release_id)) { + throw new DataContractError("data_manifest.release_id is invalid."); + } + if (!Array.isArray(manifest.model_versions) || manifest.model_versions.length === 0) { + throw new DataContractError("data_manifest.model_versions must be a non-empty array."); + } + manifest.model_versions.forEach((version, index) => + requireSafeModelVersion(version, `data_manifest.model_versions[${index}]`), + ); + if ( + JSON.stringify(manifest.model_versions) !== + JSON.stringify([...new Set(manifest.model_versions)].sort()) + ) { + throw new DataContractError("data_manifest.model_versions must be sorted and unique."); + } + requireExactKeys( + manifest.data_scope, + ["first_year", "last_year", "model_names", "partitions"], + "data_manifest.data_scope", + ); + requireInteger(manifest.data_scope.first_year, "data_manifest.data_scope.first_year", { + min: 2010, + max: 2100, + }); + requireInteger(manifest.data_scope.last_year, "data_manifest.data_scope.last_year", { + min: manifest.data_scope.first_year, + max: 2100, + }); + validateStringArray(manifest.data_scope.model_names, "data_manifest.data_scope.model_names"); + requireExactStringSet( + manifest.data_scope.partitions, + APPROVED_PARTITIONS, + "data_manifest.data_scope.partitions", + ); + requireExactKeys( + manifest.definitions, + [ + "episode_gap_days", + "locked_test_metrics_published", + "support_rounding", + "suppression_min_nonpass", + "suppression_min_pass", + "suppression_min_support", + "suppression_min_distinct_vehicles", + "suppression_min_distinct_pass_vehicles", + "suppression_min_distinct_nonpass_vehicles", + "target", + ], + "data_manifest.definitions", + ); + requireInteger(manifest.definitions.episode_gap_days, "data_manifest.definitions.episode_gap_days", { + min: 1, + }); + requireBoolean( + manifest.definitions.locked_test_metrics_published, + "data_manifest.definitions.locked_test_metrics_published", + ); + if (manifest.definitions.locked_test_metrics_published !== false) { + throw new DataContractError("Locked-test metrics are not approved for this public preview."); + } + for (const key of [ + "support_rounding", + "suppression_min_nonpass", + "suppression_min_pass", + "suppression_min_support", + "suppression_min_distinct_vehicles", + "suppression_min_distinct_pass_vehicles", + "suppression_min_distinct_nonpass_vehicles", + ]) { + requireInteger(manifest.definitions[key], `data_manifest.definitions.${key}`, { min: 1 }); + } + requireString(manifest.definitions.target, "data_manifest.definitions.target"); + if (!Array.isArray(manifest.assets)) { + throw new DataContractError("data_manifest.assets must be an array."); + } + validateStringArray(manifest.assets, "data_manifest.assets"); + const expectedManifestAssets = Object.entries(REQUIRED_ASSETS) + .filter(([name]) => !["manifest", "shaManifest"].includes(name)) + .map(([, filename]) => filename) + .sort(); + const declaredAssets = [...manifest.assets].sort(); + if (JSON.stringify(declaredAssets) !== JSON.stringify(expectedManifestAssets)) { + throw new DataContractError("data_manifest.assets does not match the approved asset set."); + } + + const validated = { manifest }; + for (const [name, validator] of Object.entries(ASSET_VALIDATORS)) { + if (!Object.hasOwn(rawAssets, name)) { + throw new DataContractError(`A required approved aggregate is missing: ${name}.`); + } + validated[name] = validator(rawAssets[name]); + if ( + validated[name].development_preview !== manifest.development_preview || + validated[name].population_estimate_allowed !== manifest.population_estimate_allowed + ) { + throw new DataContractError(`${name} publication flags disagree with data_manifest.`); + } + } + requireExactStringSet( + validated.filters.models, + manifest.data_scope.model_names, + "filter_catalog.models", + ); + for (const [index, row] of validated.diagnostics.rows.entries()) { + if (!validated.filters.models.includes(row.model)) { + throw new DataContractError(`model_diagnostics.rows[${index}].model is not cataloged.`); + } + } + const minimumSupport = manifest.definitions.suppression_min_support; + const supportRounding = manifest.definitions.support_rounding; + for (const name of ["overview", "ageRisk", "scorecards", "coverage"]) { + for (const [index, row] of validated[name].rows.entries()) { + if ( + row.support_rounded < minimumSupport || + row.support_rounded % supportRounding !== 0 + ) { + throw new DataContractError( + `${name}.rows[${index}].support_rounded violates the publication thresholds.`, + ); + } + } + } + return Object.freeze(validated); +} + +async function fetchBytes(path, fetchImplementation) { + let response; + try { + response = await fetchImplementation(path, { + cache: "no-store", + credentials: "same-origin", + }); + } catch { + throw new DataContractError("Approved dashboard assets could not be reached."); + } + if (!response.ok) { + throw new DataContractError("One or more approved dashboard assets are unavailable."); + } + try { + return await response.arrayBuffer(); + } catch { + throw new DataContractError("An approved dashboard asset could not be read safely."); + } +} + +function parseJsonBytes(bytes, label) { + if (typeof TextDecoder !== "function") { + throw new DataContractError("This browser cannot decode dashboard assets safely."); + } + try { + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + } catch { + throw new DataContractError(`${label} is not valid UTF-8 JSON.`); + } +} + +async function sha256Hex(bytes, cryptoImplementation) { + if (!cryptoImplementation?.subtle || typeof cryptoImplementation.subtle.digest !== "function") { + throw new DataContractError("This browser cannot verify dashboard asset integrity."); + } + let digest; + try { + digest = await cryptoImplementation.subtle.digest("SHA-256", bytes); + } catch { + throw new DataContractError("Dashboard asset integrity verification failed."); + } + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +export async function loadDashboardData({ + basePath = "./public/data/", + fetchImplementation = globalThis.fetch, + cryptoImplementation = globalThis.crypto, +} = {}) { + if (typeof fetchImplementation !== "function") { + throw new DataContractError("This browser cannot load dashboard assets safely."); + } + const baseUrl = new URL(basePath, globalThis.location?.href ?? "http://local/"); + const shaBytes = await fetchBytes( + new URL(REQUIRED_ASSETS.shaManifest, baseUrl), + fetchImplementation, + ); + const shaManifest = validateShaManifest( + parseJsonBytes(shaBytes, "The checksum manifest"), + ); + const expectedDigests = new Map( + shaManifest.files.map((entry) => [entry.name, entry.sha256]), + ); + const assetEntries = Object.entries(REQUIRED_ASSETS).filter( + ([name]) => name !== "shaManifest", + ); + const rawEntries = await Promise.all( + assetEntries.map(async ([name, filename]) => [ + name, + filename, + await fetchBytes(new URL(filename, baseUrl), fetchImplementation), + ]), + ); + + await Promise.all( + rawEntries.map(async ([, filename, bytes]) => { + const observed = await sha256Hex(bytes, cryptoImplementation); + if (observed !== expectedDigests.get(filename)) { + throw new DataContractError(`Integrity verification failed for ${filename}.`); + } + }), + ); + + const parsedEntries = rawEntries.map(([name, filename, bytes]) => [ + name, + parseJsonBytes(bytes, filename), + ]); + parsedEntries.push(["shaManifest", shaManifest]); + return validateAssetSet(Object.fromEntries(parsedEntries)); +} + +export { SCHEMA_VERSION }; diff --git a/dashboard/package.json b/dashboard/package.json new file mode 100644 index 0000000..6160fc8 --- /dev/null +++ b/dashboard/package.json @@ -0,0 +1,14 @@ +{ + "name": "utah-vehicle-health-dashboard", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Dependency-free static dashboard for approved Utah Vehicle Health aggregates.", + "scripts": { + "start": "node server.mjs", + "test": "node --test tests/contract.test.mjs" + }, + "engines": { + "node": ">=20" + } +} diff --git a/dashboard/public/data/age_risk_curve.json b/dashboard/public/data/age_risk_curve.json new file mode 100644 index 0000000..319c5ad --- /dev/null +++ b/dashboard/public/data/age_risk_curve.json @@ -0,0 +1 @@ +{"development_preview":true,"population_estimate_allowed":false,"rows":[{"age_band":"0-3","nonpass_rate":0.058,"support_rounded":500},{"age_band":"4-7","nonpass_rate":0.06,"support_rounded":5900},{"age_band":"8-11","nonpass_rate":0.077,"support_rounded":9800},{"age_band":"12-15","nonpass_rate":0.109,"support_rounded":9700},{"age_band":"16-20","nonpass_rate":0.157,"support_rounded":8600},{"age_band":"21+","nonpass_rate":0.221,"support_rounded":4700}],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/cohort_scorecard.json b/dashboard/public/data/cohort_scorecard.json new file mode 100644 index 0000000..a09c894 --- /dev/null +++ b/dashboard/public/data/cohort_scorecard.json @@ -0,0 +1 @@ +{"development_preview":true,"population_estimate_allowed":false,"rows":[{"nonpass_rate":0.116,"prior_make":"FORD","prior_model":"F150","support_rounded":1100},{"nonpass_rate":0.09,"prior_make":"HONDA","prior_model":"ACCORD","support_rounded":400},{"nonpass_rate":0.073,"prior_make":"HONDA","prior_model":"CIVIC","support_rounded":300},{"nonpass_rate":0.086,"prior_make":"TOYOTA","prior_model":"CAMRY","support_rounded":500}],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/coverage_quality.json b/dashboard/public/data/coverage_quality.json new file mode 100644 index 0000000..d53b6c3 --- /dev/null +++ b/dashboard/public/data/coverage_quality.json @@ -0,0 +1 @@ +{"development_preview":true,"population_estimate_allowed":false,"rows":[{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3100,"utah_obd_proxy_share":0.0,"year":2016},{"labeled_rate":0.883,"overall_result_share":0.002,"source_era":"utah","support_rounded":600,"utah_obd_proxy_share":0.998,"year":2016},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":700,"utah_obd_proxy_share":0.0,"year":2016},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3200,"utah_obd_proxy_share":0.0,"year":2017},{"labeled_rate":0.899,"overall_result_share":0.0,"source_era":"utah","support_rounded":900,"utah_obd_proxy_share":1.0,"year":2017},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":700,"utah_obd_proxy_share":0.0,"year":2017},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3200,"utah_obd_proxy_share":0.0,"year":2018},{"labeled_rate":0.921,"overall_result_share":0.001,"source_era":"utah","support_rounded":1000,"utah_obd_proxy_share":0.999,"year":2018},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2018},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3300,"utah_obd_proxy_share":0.0,"year":2019},{"labeled_rate":0.93,"overall_result_share":0.001,"source_era":"utah","support_rounded":800,"utah_obd_proxy_share":0.999,"year":2019},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2019},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3200,"utah_obd_proxy_share":0.0,"year":2020},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2020},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":2200,"utah_obd_proxy_share":0.0,"year":2021},{"labeled_rate":0.931,"overall_result_share":0.0,"source_era":"utah","support_rounded":1100,"utah_obd_proxy_share":1.0,"year":2021},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2021},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3000,"utah_obd_proxy_share":0.0,"year":2022},{"labeled_rate":0.927,"overall_result_share":0.001,"source_era":"utah","support_rounded":1100,"utah_obd_proxy_share":0.999,"year":2022},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2022},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3000,"utah_obd_proxy_share":0.0,"year":2023},{"labeled_rate":0.93,"overall_result_share":0.0,"source_era":"utah","support_rounded":1200,"utah_obd_proxy_share":1.0,"year":2023},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2023},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":500,"utah_obd_proxy_share":0.0,"year":2024},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slco","support_rounded":300,"utah_obd_proxy_share":0.0,"year":2024},{"labeled_rate":0.929,"overall_result_share":0.0,"source_era":"utah","support_rounded":1100,"utah_obd_proxy_share":1.0,"year":2024},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2024}],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/data_manifest.json b/dashboard/public/data/data_manifest.json new file mode 100644 index 0000000..44bf720 --- /dev/null +++ b/dashboard/public/data/data_manifest.json @@ -0,0 +1 @@ +{"assets":["age_risk_curve.json","cohort_scorecard.json","coverage_quality.json","filter_catalog.json","model_diagnostics.json","overview_period_county.json"],"data_scope":{"first_year":2016,"last_year":2024,"model_names":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"]},"definitions":{"episode_gap_days":30,"locked_test_metrics_published":false,"support_rounding":100,"suppression_min_distinct_nonpass_vehicles":10,"suppression_min_distinct_pass_vehicles":10,"suppression_min_distinct_vehicles":100,"suppression_min_nonpass":10,"suppression_min_pass":10,"suppression_min_support":100,"target":"first-attempt next-episode binary non-pass rate"},"development_preview":true,"model_versions":["baseline_v1","hist_gradient_boosting_v1"],"population_estimate_allowed":false,"release_id":"7c1af8d22d3841b84d2cf4cd5ba7bce6a43578c78bc3d32a25d200b4b1986704","schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/filter_catalog.json b/dashboard/public/data/filter_catalog.json new file mode 100644 index 0000000..83721a2 --- /dev/null +++ b/dashboard/public/data/filter_catalog.json @@ -0,0 +1 @@ +{"age_bands":["0-3","4-7","8-11","12-15","16-20","21+"],"development_preview":true,"models":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"],"periods":[{"quarters":[1,2,3,4],"year":2016},{"quarters":[1,2,3,4],"year":2017},{"quarters":[1,2,3,4],"year":2018},{"quarters":[1,2,3,4],"year":2019},{"quarters":[1,2,3,4],"year":2020},{"quarters":[1,2,3,4],"year":2021},{"quarters":[1,2,3,4],"year":2022},{"quarters":[1,2,3,4],"year":2023},{"quarters":[1,2,3,4],"year":2024}],"population_estimate_allowed":false,"prior_make_models":[{"prior_make":"FORD","prior_model":"F150"},{"prior_make":"HONDA","prior_model":"ACCORD"},{"prior_make":"HONDA","prior_model":"CIVIC"},{"prior_make":"TOYOTA","prior_model":"CAMRY"}],"public_counties":["salt_lake","utah","weber"],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/model_diagnostics.json b/dashboard/public/data/model_diagnostics.json new file mode 100644 index 0000000..c7f2788 --- /dev/null +++ b/dashboard/public/data/model_diagnostics.json @@ -0,0 +1 @@ +{"development_preview":true,"population_estimate_allowed":false,"rows":[{"average_precision":0.275,"brier":0.0928,"log_loss":0.3218,"model":"hist_gradient_boosting_platt","partition":"calibrate","roc_auc":0.7072,"top_10_capture":0.2744},{"average_precision":0.3966,"brier":0.0877,"log_loss":0.3011,"model":"hist_gradient_boosting_raw","partition":"train","roc_auc":0.7871,"top_10_capture":0.3658},{"average_precision":0.2743,"brier":0.1011,"log_loss":0.3431,"model":"hist_gradient_boosting_raw","partition":"tune","roc_auc":0.7147,"top_10_capture":0.2635},{"average_precision":0.275,"brier":0.0928,"log_loss":0.3218,"model":"hist_gradient_boosting_raw","partition":"calibrate","roc_auc":0.7072,"top_10_capture":0.2744},{"average_precision":0.2573,"brier":0.0936,"log_loss":0.3261,"model":"logistic_platt","partition":"calibrate","roc_auc":0.6905,"top_10_capture":0.2729},{"average_precision":0.2482,"brier":0.0963,"log_loss":0.3327,"model":"logistic_raw","partition":"train","roc_auc":0.6955,"top_10_capture":0.2593},{"average_precision":0.2821,"brier":0.1012,"log_loss":0.3448,"model":"logistic_raw","partition":"tune","roc_auc":0.7072,"top_10_capture":0.2527},{"average_precision":0.2573,"brier":0.0936,"log_loss":0.3261,"model":"logistic_raw","partition":"calibrate","roc_auc":0.6905,"top_10_capture":0.2729},{"average_precision":0.146,"brier":0.1695,"log_loss":5.8429,"model":"previous_episode_literal","partition":"train","roc_auc":0.5687,"top_10_capture":0.2128},{"average_precision":0.1658,"brier":0.1723,"log_loss":5.9436,"model":"previous_episode_literal","partition":"tune","roc_auc":0.5821,"top_10_capture":0.2291},{"average_precision":0.1476,"brier":0.1677,"log_loss":5.7909,"model":"previous_episode_literal","partition":"calibrate","roc_auc":0.5767,"top_10_capture":0.224},{"average_precision":0.116,"brier":0.1025,"log_loss":0.3588,"model":"training_prevalence","partition":"train","roc_auc":0.5,"top_10_capture":0.1},{"average_precision":0.1246,"brier":0.1092,"log_loss":0.3764,"model":"training_prevalence","partition":"tune","roc_auc":0.5,"top_10_capture":0.1},{"average_precision":0.1129,"brier":0.1002,"log_loss":0.3526,"model":"training_prevalence","partition":"calibrate","roc_auc":0.5,"top_10_capture":0.1}],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/overview_period_county.json b/dashboard/public/data/overview_period_county.json new file mode 100644 index 0000000..5c3357d --- /dev/null +++ b/dashboard/public/data/overview_period_county.json @@ -0,0 +1 @@ +{"development_preview":true,"population_estimate_allowed":false,"rows":[{"nonpass_rate":0.134,"public_county":"salt_lake","quarter":1,"support_rounded":800,"year":2016},{"nonpass_rate":0.142,"public_county":"utah","quarter":1,"support_rounded":100,"year":2016},{"nonpass_rate":0.151,"public_county":"weber","quarter":1,"support_rounded":200,"year":2016},{"nonpass_rate":0.115,"public_county":"salt_lake","quarter":2,"support_rounded":800,"year":2016},{"nonpass_rate":0.078,"public_county":"utah","quarter":2,"support_rounded":200,"year":2016},{"nonpass_rate":0.118,"public_county":"weber","quarter":2,"support_rounded":200,"year":2016},{"nonpass_rate":0.103,"public_county":"salt_lake","quarter":3,"support_rounded":800,"year":2016},{"nonpass_rate":0.081,"public_county":"utah","quarter":3,"support_rounded":200,"year":2016},{"nonpass_rate":0.09,"public_county":"weber","quarter":3,"support_rounded":200,"year":2016},{"nonpass_rate":0.116,"public_county":"salt_lake","quarter":4,"support_rounded":700,"year":2016},{"nonpass_rate":0.092,"public_county":"utah","quarter":4,"support_rounded":100,"year":2016},{"nonpass_rate":0.132,"public_county":"weber","quarter":4,"support_rounded":100,"year":2016},{"nonpass_rate":0.104,"public_county":"salt_lake","quarter":1,"support_rounded":800,"year":2017},{"nonpass_rate":0.075,"public_county":"utah","quarter":1,"support_rounded":200,"year":2017},{"nonpass_rate":0.119,"public_county":"weber","quarter":1,"support_rounded":200,"year":2017},{"nonpass_rate":0.117,"public_county":"salt_lake","quarter":2,"support_rounded":900,"year":2017},{"nonpass_rate":0.075,"public_county":"utah","quarter":2,"support_rounded":200,"year":2017},{"nonpass_rate":0.121,"public_county":"weber","quarter":2,"support_rounded":200,"year":2017},{"nonpass_rate":0.103,"public_county":"salt_lake","quarter":3,"support_rounded":900,"year":2017},{"nonpass_rate":0.079,"public_county":"utah","quarter":3,"support_rounded":200,"year":2017},{"nonpass_rate":0.137,"public_county":"weber","quarter":3,"support_rounded":200,"year":2017},{"nonpass_rate":0.123,"public_county":"salt_lake","quarter":4,"support_rounded":700,"year":2017},{"nonpass_rate":0.123,"public_county":"utah","quarter":4,"support_rounded":100,"year":2017},{"nonpass_rate":0.066,"public_county":"weber","quarter":4,"support_rounded":200,"year":2017},{"nonpass_rate":0.102,"public_county":"salt_lake","quarter":1,"support_rounded":800,"year":2018},{"nonpass_rate":0.088,"public_county":"utah","quarter":1,"support_rounded":200,"year":2018},{"nonpass_rate":0.138,"public_county":"weber","quarter":1,"support_rounded":200,"year":2018},{"nonpass_rate":0.116,"public_county":"salt_lake","quarter":2,"support_rounded":900,"year":2018},{"nonpass_rate":0.092,"public_county":"utah","quarter":2,"support_rounded":200,"year":2018},{"nonpass_rate":0.089,"public_county":"weber","quarter":2,"support_rounded":200,"year":2018},{"nonpass_rate":0.118,"public_county":"salt_lake","quarter":3,"support_rounded":800,"year":2018},{"nonpass_rate":0.107,"public_county":"utah","quarter":3,"support_rounded":300,"year":2018},{"nonpass_rate":0.1,"public_county":"weber","quarter":3,"support_rounded":200,"year":2018},{"nonpass_rate":0.12,"public_county":"salt_lake","quarter":4,"support_rounded":700,"year":2018},{"nonpass_rate":0.103,"public_county":"utah","quarter":4,"support_rounded":200,"year":2018},{"nonpass_rate":0.091,"public_county":"weber","quarter":4,"support_rounded":200,"year":2018},{"nonpass_rate":0.125,"public_county":"salt_lake","quarter":1,"support_rounded":800,"year":2019},{"nonpass_rate":0.081,"public_county":"utah","quarter":1,"support_rounded":200,"year":2019},{"nonpass_rate":0.083,"public_county":"weber","quarter":1,"support_rounded":200,"year":2019},{"nonpass_rate":0.117,"public_county":"salt_lake","quarter":2,"support_rounded":800,"year":2019},{"nonpass_rate":0.097,"public_county":"utah","quarter":2,"support_rounded":300,"year":2019},{"nonpass_rate":0.103,"public_county":"weber","quarter":2,"support_rounded":200,"year":2019},{"nonpass_rate":0.121,"public_county":"salt_lake","quarter":3,"support_rounded":900,"year":2019},{"nonpass_rate":0.092,"public_county":"utah","quarter":3,"support_rounded":200,"year":2019},{"nonpass_rate":0.115,"public_county":"weber","quarter":3,"support_rounded":200,"year":2019},{"nonpass_rate":0.121,"public_county":"salt_lake","quarter":4,"support_rounded":800,"year":2019},{"nonpass_rate":0.068,"public_county":"weber","quarter":4,"support_rounded":200,"year":2019},{"nonpass_rate":0.119,"public_county":"salt_lake","quarter":1,"support_rounded":800,"year":2020},{"nonpass_rate":0.133,"public_county":"weber","quarter":1,"support_rounded":200,"year":2020},{"nonpass_rate":0.125,"public_county":"salt_lake","quarter":2,"support_rounded":900,"year":2020},{"nonpass_rate":0.129,"public_county":"weber","quarter":2,"support_rounded":300,"year":2020},{"nonpass_rate":0.129,"public_county":"salt_lake","quarter":3,"support_rounded":900,"year":2020},{"nonpass_rate":0.146,"public_county":"weber","quarter":3,"support_rounded":200,"year":2020},{"nonpass_rate":0.147,"public_county":"salt_lake","quarter":4,"support_rounded":600,"year":2020},{"nonpass_rate":0.095,"public_county":"weber","quarter":4,"support_rounded":100,"year":2020},{"nonpass_rate":0.131,"public_county":"salt_lake","quarter":1,"support_rounded":400,"year":2021},{"nonpass_rate":0.105,"public_county":"utah","quarter":1,"support_rounded":200,"year":2021},{"nonpass_rate":0.15,"public_county":"weber","quarter":1,"support_rounded":200,"year":2021},{"nonpass_rate":0.117,"public_county":"salt_lake","quarter":2,"support_rounded":700,"year":2021},{"nonpass_rate":0.106,"public_county":"utah","quarter":2,"support_rounded":300,"year":2021},{"nonpass_rate":0.167,"public_county":"weber","quarter":2,"support_rounded":200,"year":2021},{"nonpass_rate":0.117,"public_county":"salt_lake","quarter":3,"support_rounded":500,"year":2021},{"nonpass_rate":0.077,"public_county":"utah","quarter":3,"support_rounded":300,"year":2021},{"nonpass_rate":0.162,"public_county":"weber","quarter":3,"support_rounded":300,"year":2021},{"nonpass_rate":0.133,"public_county":"salt_lake","quarter":4,"support_rounded":500,"year":2021},{"nonpass_rate":0.099,"public_county":"utah","quarter":4,"support_rounded":200,"year":2021},{"nonpass_rate":0.126,"public_county":"weber","quarter":4,"support_rounded":100,"year":2021},{"nonpass_rate":0.125,"public_county":"salt_lake","quarter":1,"support_rounded":700,"year":2022},{"nonpass_rate":0.063,"public_county":"utah","quarter":1,"support_rounded":300,"year":2022},{"nonpass_rate":0.131,"public_county":"weber","quarter":1,"support_rounded":200,"year":2022},{"nonpass_rate":0.143,"public_county":"salt_lake","quarter":2,"support_rounded":700,"year":2022},{"nonpass_rate":0.087,"public_county":"utah","quarter":2,"support_rounded":300,"year":2022},{"nonpass_rate":0.134,"public_county":"weber","quarter":2,"support_rounded":200,"year":2022},{"nonpass_rate":0.136,"public_county":"salt_lake","quarter":3,"support_rounded":800,"year":2022},{"nonpass_rate":0.076,"public_county":"utah","quarter":3,"support_rounded":300,"year":2022},{"nonpass_rate":0.134,"public_county":"weber","quarter":3,"support_rounded":200,"year":2022},{"nonpass_rate":0.132,"public_county":"salt_lake","quarter":4,"support_rounded":700,"year":2022},{"nonpass_rate":0.155,"public_county":"utah","quarter":4,"support_rounded":200,"year":2022},{"nonpass_rate":0.09,"public_county":"weber","quarter":4,"support_rounded":200,"year":2022},{"nonpass_rate":0.121,"public_county":"salt_lake","quarter":1,"support_rounded":700,"year":2023},{"nonpass_rate":0.102,"public_county":"utah","quarter":1,"support_rounded":300,"year":2023},{"nonpass_rate":0.124,"public_county":"weber","quarter":1,"support_rounded":200,"year":2023},{"nonpass_rate":0.107,"public_county":"salt_lake","quarter":2,"support_rounded":800,"year":2023},{"nonpass_rate":0.11,"public_county":"utah","quarter":2,"support_rounded":300,"year":2023},{"nonpass_rate":0.141,"public_county":"weber","quarter":2,"support_rounded":200,"year":2023},{"nonpass_rate":0.138,"public_county":"salt_lake","quarter":3,"support_rounded":800,"year":2023},{"nonpass_rate":0.095,"public_county":"utah","quarter":3,"support_rounded":300,"year":2023},{"nonpass_rate":0.178,"public_county":"weber","quarter":3,"support_rounded":200,"year":2023},{"nonpass_rate":0.137,"public_county":"salt_lake","quarter":4,"support_rounded":700,"year":2023},{"nonpass_rate":0.097,"public_county":"utah","quarter":4,"support_rounded":200,"year":2023},{"nonpass_rate":0.1,"public_county":"weber","quarter":4,"support_rounded":200,"year":2023},{"nonpass_rate":0.124,"public_county":"salt_lake","quarter":1,"support_rounded":500,"year":2024},{"nonpass_rate":0.105,"public_county":"utah","quarter":1,"support_rounded":200,"year":2024},{"nonpass_rate":0.09,"public_county":"weber","quarter":1,"support_rounded":200,"year":2024},{"nonpass_rate":0.123,"public_county":"utah","quarter":2,"support_rounded":300,"year":2024},{"nonpass_rate":0.105,"public_county":"weber","quarter":2,"support_rounded":200,"year":2024},{"nonpass_rate":0.096,"public_county":"utah","quarter":3,"support_rounded":300,"year":2024},{"nonpass_rate":0.15,"public_county":"weber","quarter":3,"support_rounded":200,"year":2024},{"nonpass_rate":0.094,"public_county":"salt_lake","quarter":4,"support_rounded":300,"year":2024},{"nonpass_rate":0.128,"public_county":"utah","quarter":4,"support_rounded":200,"year":2024},{"nonpass_rate":0.098,"public_county":"weber","quarter":4,"support_rounded":200,"year":2024}],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/sha256_manifest.json b/dashboard/public/data/sha256_manifest.json new file mode 100644 index 0000000..86b1e1e --- /dev/null +++ b/dashboard/public/data/sha256_manifest.json @@ -0,0 +1 @@ +{"development_preview":true,"files":[{"name":"age_risk_curve.json","sha256":"5445d52365ad3494ba05f097e7f4f5e41fffeecfe50fbb2952d72044f496c925"},{"name":"cohort_scorecard.json","sha256":"2ae83d9d55aa1f2ccae6420c66350fb491405eedfa90374aa3df2782a4aa0bf1"},{"name":"coverage_quality.json","sha256":"35b7dbed50e1b8260aac18c269f5045010ea23649a649f984a4693f3f00a1e83"},{"name":"data_manifest.json","sha256":"b7196b04eb223683c928cfb5375ed618e1230fa03382bd8a889de24c589557a0"},{"name":"filter_catalog.json","sha256":"9e45d428cd38853002ebd0bda089eb46d0832cd1b7b6b05ac953bc8472b01eb5"},{"name":"model_diagnostics.json","sha256":"ebf99eb32436a5eabb7d754cd88109f2cc0981575dd28966757031ac7faafa57"},{"name":"overview_period_county.json","sha256":"fe2a4233563b47fa31fc87859b100983d3bfb574c5a7fbc7241b9cfb4661b038"}],"population_estimate_allowed":false,"schema_version":"dashboard_data_v1"} diff --git a/dashboard/server.mjs b/dashboard/server.mjs new file mode 100644 index 0000000..4cadc66 --- /dev/null +++ b/dashboard/server.mjs @@ -0,0 +1,82 @@ +import { createReadStream, realpathSync, statSync } from "node:fs"; +import { createServer } from "node:http"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = realpathSync(path.dirname(fileURLToPath(import.meta.url))); +const HOST = process.env.HOST || "127.0.0.1"; +const PORT = Number.parseInt(process.env.PORT || "4173", 10); +const TYPES = new Map([ + [".css", "text/css; charset=utf-8"], + [".html", "text/html; charset=utf-8"], + [".js", "text/javascript; charset=utf-8"], + [".json", "application/json; charset=utf-8"], + [".mjs", "text/javascript; charset=utf-8"], + [".svg", "image/svg+xml"], +]); +const PUBLIC_PATHS = new Set([ + "/index.html", + "/styles.css", + "/js/app.js", + "/js/charts.js", + "/js/data.js", + "/public/data/age_risk_curve.json", + "/public/data/cohort_scorecard.json", + "/public/data/coverage_quality.json", + "/public/data/data_manifest.json", + "/public/data/filter_catalog.json", + "/public/data/model_diagnostics.json", + "/public/data/overview_period_county.json", + "/public/data/sha256_manifest.json", +]); + +function safeFile(requestUrl) { + let pathname; + try { + pathname = decodeURIComponent(new URL(requestUrl, "http://local").pathname); + } catch { + return { file: null, status: 400 }; + } + if (pathname === "/") pathname = "/index.html"; + if (!PUBLIC_PATHS.has(pathname)) return { file: null, status: 404 }; + try { + const candidate = realpathSync(path.join(ROOT, pathname.slice(1))); + if (!candidate.startsWith(`${ROOT}${path.sep}`) || !statSync(candidate).isFile()) { + return { file: null, status: 404 }; + } + return { file: candidate, status: 200 }; + } catch { + return { file: null, status: 404 }; + } +} + +const server = createServer((request, response) => { + if (!request.url || !["GET", "HEAD"].includes(request.method || "")) { + response.writeHead(405, { Allow: "GET, HEAD" }); + response.end("Method not allowed"); + return; + } + const resolved = safeFile(request.url); + if (!resolved.file) { + response.writeHead(resolved.status, { "Content-Type": "text/plain; charset=utf-8" }); + response.end(resolved.status === 400 ? "Bad request" : "Not found"); + return; + } + const file = resolved.file; + const extension = path.extname(file).toLowerCase(); + response.writeHead(200, { + "Content-Type": TYPES.get(extension) || "application/octet-stream", + "Cache-Control": extension === ".json" ? "no-store" : "public, max-age=300", + "X-Content-Type-Options": "nosniff", + "Referrer-Policy": "no-referrer", + }); + if (request.method === "HEAD") { + response.end(); + return; + } + createReadStream(file).on("error", () => response.destroy()).pipe(response); +}); + +server.listen(PORT, HOST, () => { + process.stdout.write(`Utah Vehicle Health dashboard: http://${HOST}:${PORT}\n`); +}); diff --git a/dashboard/styles.css b/dashboard/styles.css new file mode 100644 index 0000000..c865326 --- /dev/null +++ b/dashboard/styles.css @@ -0,0 +1,1358 @@ +:root { + color-scheme: light; + --ink-950: #172b2b; + --ink-800: #294342; + --ink-650: #49615f; + --ink-500: #6d817f; + --paper: #fffdf8; + --paper-warm: #faf5e9; + --sand-100: #f4ead4; + --sand-200: #e7d8b9; + --sand-400: #c5a66c; + --teal-800: #155e60; + --teal-700: #197477; + --teal-600: #23888a; + --teal-100: #dcefee; + --red-700: #a63d32; + --red-100: #f7e3df; + --amber-700: #a66516; + --amber-100: #f9eccd; + --purple-700: #704b87; + --purple-100: #eee4f3; + --gray-100: #edf0ed; + --gray-300: #cbd3d0; + --white: #ffffff; + --shadow-sm: 0 1px 2px rgb(23 43 43 / 7%), 0 7px 24px rgb(23 43 43 / 5%); + --shadow-md: 0 18px 48px rgb(23 43 43 / 10%); + --radius-sm: 0.55rem; + --radius-md: 0.9rem; + --radius-lg: 1.35rem; + --content: 1180px; + --font-sans: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + --font-display: Georgia, "Times New Roman", serif; +} + +* { + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +body { + min-width: 320px; + margin: 0; + color: var(--ink-950); + background: + radial-gradient(circle at 7% 2%, rgb(231 216 185 / 35%), transparent 30rem), + var(--paper); + font-family: var(--font-sans); + font-size: 1rem; + line-height: 1.55; + -webkit-font-smoothing: antialiased; +} + +button, +input, +select { + font: inherit; +} + +button, +select { + cursor: pointer; +} + +button:disabled, +select:disabled, +input:disabled { + cursor: not-allowed; +} + +a { + color: var(--teal-800); +} + +code { + padding: 0.08em 0.28em; + border-radius: 0.3em; + background: var(--gray-100); + font-size: 0.9em; +} + +h1, +h2, +h3, +p { + margin-top: 0; +} + +h1, +h2, +h3 { + line-height: 1.16; +} + +h1 { + max-width: 18ch; + margin-bottom: 1rem; + font-family: var(--font-display); + font-size: clamp(2.35rem, 6vw, 4.4rem); + font-weight: 500; + letter-spacing: -0.035em; +} + +h2 { + margin-bottom: 0.7rem; + font-size: clamp(1.15rem, 2vw, 1.45rem); + letter-spacing: -0.018em; +} + +.skip-link { + position: fixed; + z-index: 100; + top: 0.75rem; + left: 0.75rem; + padding: 0.75rem 1rem; + transform: translateY(-200%); + border-radius: var(--radius-sm); + color: var(--white); + background: var(--ink-950); + font-weight: 700; +} + +.skip-link:focus { + transform: translateY(0); +} + +:focus-visible { + outline: 3px solid var(--amber-700); + outline-offset: 3px; +} + +.sr-only { + position: absolute !important; + width: 1px !important; + height: 1px !important; + padding: 0 !important; + margin: -1px !important; + overflow: hidden !important; + clip: rect(0, 0, 0, 0) !important; + white-space: nowrap !important; + border: 0 !important; +} + +.preview-banner { + position: relative; + z-index: 20; + display: flex; + justify-content: center; + gap: 0.65rem; + padding: 0.58rem 1rem; + border-bottom: 1px solid #e9c77d; + color: #503407; + background: var(--amber-100); + font-size: 0.86rem; + text-align: center; +} + +.preview-banner[hidden] { + display: none; +} + +.preview-banner strong { + margin-right: 0.35rem; +} + +.preview-banner__icon { + color: var(--amber-700); +} + +.site-header { + position: sticky; + z-index: 10; + top: 0; + border-bottom: 1px solid rgb(197 166 108 / 30%); + background: rgb(255 253 248 / 93%); + box-shadow: 0 3px 16px rgb(23 43 43 / 4%); + backdrop-filter: blur(16px); +} + +.site-header__inner, +.primary-nav__inner, +.page-shell { + width: min(calc(100% - 2rem), var(--content)); + margin-inline: auto; +} + +.site-header__inner { + display: flex; + align-items: center; + justify-content: space-between; + gap: 2rem; + min-height: 78px; +} + +.brand { + display: inline-flex; + align-items: center; + gap: 0.8rem; + color: var(--ink-950); + text-decoration: none; +} + +.brand__mark { + width: 44px; + height: 44px; + flex: 0 0 auto; + color: var(--teal-800); +} + +.brand__name, +.brand__tagline { + display: block; +} + +.brand__name { + font-family: var(--font-display); + font-size: 1.22rem; + font-weight: 700; + letter-spacing: -0.015em; +} + +.brand__tagline { + margin-top: 0.05rem; + color: var(--ink-650); + font-size: 0.72rem; + letter-spacing: 0.07em; + text-transform: uppercase; +} + +.header-status { + display: flex; + align-items: center; + gap: 0.55rem; + color: var(--ink-650); + font-size: 0.8rem; + font-weight: 650; +} + +.status-dot { + width: 0.6rem; + height: 0.6rem; + border-radius: 50%; + background: var(--amber-700); + box-shadow: 0 0 0 4px var(--amber-100); +} + +.header-status[data-state="ready"] .status-dot { + background: var(--teal-700); + box-shadow: 0 0 0 4px var(--teal-100); +} + +.header-status[data-state="error"] .status-dot { + background: var(--red-700); + box-shadow: 0 0 0 4px var(--red-100); +} + +.primary-nav { + border-top: 1px solid rgb(197 166 108 / 22%); +} + +.primary-nav__inner { + display: flex; + gap: 0.2rem; + overflow-x: auto; + scrollbar-width: thin; +} + +.primary-nav a { + position: relative; + flex: 0 0 auto; + padding: 0.9rem 1rem 0.82rem; + color: var(--ink-650); + font-size: 0.88rem; + font-weight: 700; + text-decoration: none; +} + +.primary-nav a::after { + position: absolute; + right: 1rem; + bottom: 0; + left: 1rem; + height: 3px; + border-radius: 3px 3px 0 0; + background: transparent; + content: ""; +} + +.primary-nav a:hover { + color: var(--teal-800); + background: rgb(220 239 238 / 40%); +} + +.primary-nav a[aria-current="page"] { + color: var(--teal-800); +} + +.primary-nav a[aria-current="page"]::after { + background: var(--teal-700); +} + +main { + min-height: 65vh; +} + +.view { + padding: clamp(2.5rem, 6vw, 5rem) 0 5rem; +} + +.view[hidden] { + display: none; +} + +.page-shell--narrow { + max-width: 940px; +} + +.page-heading { + margin-bottom: clamp(2rem, 5vw, 3.4rem); +} + +.page-heading--split { + display: flex; + align-items: end; + justify-content: space-between; + gap: 3rem; +} + +.eyebrow { + margin-bottom: 0.45rem; + color: var(--teal-800); + font-size: 0.73rem; + font-weight: 800; + letter-spacing: 0.14em; + text-transform: uppercase; +} + +.lede { + max-width: 70ch; + margin-bottom: 0; + color: var(--ink-650); + font-size: clamp(1.02rem, 2vw, 1.17rem); +} + +.data-stamp { + min-width: 190px; + padding: 1rem 1.1rem; + border: 1px solid var(--sand-200); + border-radius: var(--radius-md); + background: rgb(255 255 255 / 70%); + box-shadow: var(--shadow-sm); +} + +.data-stamp span, +.data-stamp strong, +.data-stamp small { + display: block; +} + +.data-stamp span, +.data-stamp small { + color: var(--ink-650); + font-size: 0.76rem; +} + +.data-stamp strong { + margin: 0.22rem 0; + font-size: 1.04rem; +} + +.unavailable-state { + display: flex; + align-items: flex-start; + gap: 1rem; + margin: 0 0 2rem; + padding: 1.25rem; + border: 1px solid #e7b7b1; + border-radius: var(--radius-md); + background: var(--red-100); +} + +.unavailable-state[hidden] { + display: none; +} + +.unavailable-state h2, +.unavailable-state p { + margin-bottom: 0.25rem; +} + +.unavailable-state__icon { + display: grid; + width: 2rem; + height: 2rem; + flex: 0 0 auto; + place-items: center; + border-radius: 50%; + color: var(--white); + background: var(--red-700); + font-weight: 900; +} + +.kpi-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 1rem; + margin-bottom: 1rem; +} + +.kpi-card { + position: relative; + min-height: 155px; + padding: 1.25rem; + overflow: hidden; + border: 1px solid var(--sand-200); + border-radius: var(--radius-md); + background: rgb(255 255 255 / 80%); + box-shadow: var(--shadow-sm); +} + +.kpi-card::before { + position: absolute; + top: 0; + right: 0; + left: 0; + height: 4px; + background: var(--sand-400); + content: ""; +} + +.kpi-card--pass::before { + background: var(--teal-700); +} + +.kpi-card--nonpass::before { + background: var(--red-700); +} + +.kpi-card__label, +.kpi-card strong, +.kpi-card small { + display: block; +} + +.kpi-card__label { + color: var(--ink-650); + font-size: 0.8rem; + font-weight: 750; +} + +.kpi-card strong { + margin: 0.45rem 0 0.25rem; + font-family: var(--font-display); + font-size: clamp(2rem, 4vw, 2.85rem); + font-weight: 500; + line-height: 1; +} + +.kpi-card small { + color: var(--ink-500); + font-size: 0.73rem; +} + +.dashboard-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 1rem; + margin: 1rem 0; +} + +.dashboard-grid--wide { + grid-template-columns: minmax(0, 1.45fr) minmax(290px, 0.75fr); +} + +.panel { + min-width: 0; + padding: clamp(1.15rem, 2.5vw, 1.55rem); + border: 1px solid var(--sand-200); + border-radius: var(--radius-lg); + background: rgb(255 255 255 / 86%); + box-shadow: var(--shadow-sm); +} + +.panel--wide { + grid-column: 1 / -1; +} + +.panel__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 1rem; +} + +.panel__header h2, +.panel__header p { + margin-bottom: 0; +} + +.chart-legend { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 0.5rem 0.9rem; + color: var(--ink-650); + font-size: 0.7rem; +} + +.chart-legend span { + display: inline-flex; + align-items: center; + gap: 0.35rem; +} + +.legend-swatch { + width: 0.62rem; + height: 0.62rem; + border-radius: 0.18rem; + background: var(--gray-300); +} + +.legend-swatch--pass { background: var(--teal-700); } +.legend-swatch--fail { background: var(--red-700); } +.legend-swatch--reject { background: var(--amber-700); } +.legend-swatch--abort { background: var(--purple-700); } +.legend-swatch--missing { background: var(--gray-300); } + +.chart { + min-height: 260px; +} + +.chart--large { + min-height: 330px; +} + +.chart--scorecard { + min-height: 360px; +} + +.chart svg, +.utah-map svg { + display: block; + width: 100%; + height: auto; + overflow: visible; +} + +.chart .axis-line, +.chart .grid-line { + stroke: var(--gray-300); + vector-effect: non-scaling-stroke; +} + +.chart .grid-line { + stroke-dasharray: 3 4; + opacity: 0.7; +} + +.chart .axis-label { + fill: var(--ink-650); + font-family: var(--font-sans); + font-size: 11px; +} + +.chart .chart-line { + fill: none; + stroke: var(--red-700); + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 3; + vector-effect: non-scaling-stroke; +} + +.chart .chart-dot { + fill: var(--paper); + stroke: var(--red-700); + stroke-width: 2.5; + vector-effect: non-scaling-stroke; +} + +.chart .confidence-line { + stroke: var(--ink-500); + stroke-width: 1.5; + vector-effect: non-scaling-stroke; +} + +.chart .reference-line { + stroke: var(--ink-500); + stroke-dasharray: 6 4; + vector-effect: non-scaling-stroke; +} + +.chart-note { + margin: 0.7rem 0 0; + color: var(--ink-650); + font-size: 0.75rem; +} + +.chart-empty { + display: grid; + min-height: 240px; + place-items: center; + border: 1px dashed var(--gray-300); + border-radius: var(--radius-md); + color: var(--ink-650); + background: var(--gray-100); + text-align: center; +} + +.utah-map { + max-width: 250px; + margin: 0 auto 1rem; +} + +.county-list { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; +} + +.county-chip { + padding: 0.28rem 0.5rem; + border: 1px solid var(--gray-300); + border-radius: 999px; + color: var(--ink-500); + background: var(--gray-100); + font-size: 0.65rem; +} + +.county-chip[data-covered="true"] { + border-color: #9bc9c6; + color: var(--teal-800); + background: var(--teal-100); + font-weight: 700; +} + +.callout { + display: flex; + align-items: flex-start; + gap: 0.8rem; + margin-top: 1rem; + padding: 1rem 1.15rem; + border-left: 4px solid var(--teal-700); + border-radius: 0 var(--radius-sm) var(--radius-sm) 0; + background: var(--teal-100); +} + +.callout p { + margin: 0; + font-size: 0.86rem; +} + +.callout__icon { + display: grid; + width: 1.4rem; + height: 1.4rem; + flex: 0 0 auto; + place-items: center; + border-radius: 50%; + color: var(--white); + background: var(--teal-800); + font-size: 0.75rem; + font-weight: 800; +} + +.explorer-layout { + display: grid; + grid-template-columns: 250px minmax(0, 1fr); + gap: 1.25rem; + align-items: start; +} + +.filter-panel { + position: sticky; + top: 155px; + padding: 1.2rem; + border: 1px solid var(--sand-200); + border-radius: var(--radius-lg); + background: var(--paper-warm); +} + +.filter-panel__heading, +.result-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; +} + +.filter-panel__heading h2 { + margin: 0; +} + +.text-button { + padding: 0.2rem; + border: 0; + color: var(--teal-800); + background: transparent; + font-size: 0.75rem; + font-weight: 750; + text-decoration: underline; + text-underline-offset: 0.18em; +} + +.field { + display: grid; + gap: 0.35rem; + margin-top: 0.9rem; +} + +.field > span, +.segmented-control legend { + color: var(--ink-650); + font-size: 0.73rem; + font-weight: 750; +} + +.field input, +.field select { + width: 100%; + min-height: 42px; + padding: 0.58rem 0.68rem; + border: 1px solid var(--gray-300); + border-radius: var(--radius-sm); + color: var(--ink-950); + background: var(--white); + font-size: 0.83rem; +} + +.field input:disabled, +.field select:disabled, +fieldset:disabled .field input, +fieldset:disabled .field select { + color: var(--ink-500); + background: var(--gray-100); +} + +.field--inline { + display: flex; + align-items: center; + gap: 0.5rem; + margin: 0; +} + +.field--inline select { + width: auto; +} + +.segmented-control { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.3rem; + margin: 1rem 0 0; + padding: 0; + border: 0; +} + +.segmented-control legend { + grid-column: 1 / -1; + margin-bottom: 0.35rem; +} + +.segmented-control label { + position: relative; +} + +.segmented-control input { + position: absolute; + opacity: 0; +} + +.segmented-control span { + display: block; + padding: 0.48rem 0.4rem; + border: 1px solid var(--gray-300); + border-radius: var(--radius-sm); + color: var(--ink-650); + background: var(--white); + font-size: 0.7rem; + font-weight: 750; + text-align: center; +} + +.segmented-control input:checked + span { + border-color: var(--teal-700); + color: var(--teal-800); + background: var(--teal-100); +} + +.segmented-control input:focus-visible + span { + outline: 3px solid var(--amber-700); + outline-offset: 2px; +} + +.result-toolbar { + min-height: 48px; + margin-bottom: 0.75rem; +} + +.result-toolbar p { + margin: 0; + color: var(--ink-650); + font-size: 0.83rem; +} + +.cohort-cards { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0.75rem; + margin-top: 0.75rem; +} + +.cohort-card { + padding: 1rem; + border: 1px solid var(--sand-200); + border-radius: var(--radius-md); + background: var(--white); +} + +.cohort-card h3 { + margin-bottom: 0.2rem; + font-size: 0.95rem; +} + +.cohort-card__meta { + color: var(--ink-500); + font-size: 0.7rem; +} + +.risk-bar { + height: 0.52rem; + margin: 0.85rem 0 0.45rem; + overflow: hidden; + border-radius: 999px; + background: var(--gray-100); +} + +.risk-bar span { + display: block; + height: 100%; + border-radius: inherit; + background: var(--red-700); +} + +.cohort-card__value { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 0.5rem; +} + +.cohort-card__value strong { + font-family: var(--font-display); + font-size: 1.5rem; + font-weight: 500; +} + +.cohort-card__value small { + color: var(--ink-500); +} + +.empty-results { + padding: 2.5rem 1rem; + color: var(--ink-650); + text-align: center; +} + +.locked-panel { + display: flex; + gap: 1.2rem; + margin-bottom: 1.25rem; + padding: clamp(1.25rem, 3vw, 1.8rem); + border: 1px solid var(--sand-400); + border-radius: var(--radius-lg); + background: var(--amber-100); +} + +.locked-panel p:last-child { + margin-bottom: 0; +} + +.locked-panel__icon { + display: grid; + width: 3rem; + height: 3rem; + flex: 0 0 auto; + place-items: center; + border: 2px solid var(--amber-700); + border-radius: 50%; + color: var(--amber-700); + font-size: 1.4rem; +} + +.estimator-form { + padding: clamp(1.2rem, 3vw, 1.7rem); + border: 1px solid var(--gray-300); + border-radius: var(--radius-lg); + background: var(--white); + opacity: 0.72; +} + +.estimator-form fieldset { + padding: 0; + border: 0; +} + +.estimator-form legend { + margin-bottom: 0.2rem; + font-size: 1.05rem; + font-weight: 750; +} + +.form-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0 1rem; +} + +.button { + margin-top: 1.2rem; + padding: 0.72rem 1rem; + border: 0; + border-radius: var(--radius-sm); + color: var(--white); + background: var(--teal-800); + font-weight: 750; +} + +.form-note { + margin: 0.65rem 0 2rem; + color: var(--ink-650); + font-size: 0.78rem; +} + +.method-preview { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 2rem; + align-items: start; + padding: 1.5rem; + border-radius: var(--radius-lg); + color: var(--white); + background: var(--ink-950); +} + +.method-preview .eyebrow { + color: #9bd4d1; +} + +.check-list, +.limitation-list { + margin: 0; + padding: 0; + list-style: none; +} + +.check-list li, +.limitation-list li { + position: relative; + padding: 0.48rem 0 0.48rem 1.5rem; +} + +.check-list li::before, +.limitation-list li::before { + position: absolute; + left: 0; + content: "✓"; + color: #9bd4d1; + font-weight: 800; +} + +.method-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 1rem; + margin-bottom: 1rem; +} + +.method-card { + padding: 1.25rem; + border: 1px solid var(--sand-200); + border-radius: var(--radius-md); + background: var(--paper-warm); +} + +.method-card__number { + display: block; + margin-bottom: 1.2rem; + color: var(--sand-400); + font-family: var(--font-display); + font-size: 1.6rem; +} + +.method-card p { + margin: 0; + color: var(--ink-650); + font-size: 0.83rem; +} + +.heatmap { + min-height: 210px; + overflow-x: auto; +} + +.heatmap-grid { + display: grid; + gap: 0.3rem; + min-width: 650px; +} + +.heatmap-label { + align-self: center; + color: var(--ink-650); + font-size: 0.68rem; + font-weight: 700; +} + +.heatmap-cell { + min-height: 30px; + border: 1px solid rgb(21 94 96 / 12%); + border-radius: 0.25rem; + background: var(--gray-100); +} + +.split-section { + display: grid; + grid-template-columns: minmax(220px, 0.55fr) minmax(0, 1.45fr); + gap: 2rem; + align-items: center; + margin: 1rem 0; + padding: clamp(1.3rem, 3vw, 2rem); + border-radius: var(--radius-lg); + background: var(--paper-warm); +} + +.split-section__copy p:last-child { + margin: 0; + color: var(--ink-650); + font-size: 0.84rem; +} + +.timeline { + display: grid; + grid-template-columns: repeat(6, minmax(95px, 1fr)); + margin: 0; + padding: 0; + overflow-x: auto; + list-style: none; +} + +.timeline li { + position: relative; + min-width: 95px; + padding: 1.2rem 0.5rem 0; + border-top: 3px solid var(--teal-700); +} + +.timeline li::before { + position: absolute; + top: -7px; + left: 0; + width: 11px; + height: 11px; + border: 2px solid var(--paper-warm); + border-radius: 50%; + background: var(--teal-700); + box-shadow: 0 0 0 1px var(--teal-700); + content: ""; +} + +.timeline__holdout { + border-color: var(--purple-700) !important; +} + +.timeline__holdout::before { + background: var(--purple-700) !important; + box-shadow: 0 0 0 1px var(--purple-700) !important; +} + +.timeline span, +.timeline strong, +.timeline small { + display: block; +} + +.timeline span { + color: var(--ink-500); + font-size: 0.65rem; +} + +.timeline strong { + margin: 0.1rem 0; + font-size: 0.75rem; +} + +.timeline small { + color: var(--ink-500); + font-size: 0.58rem; + line-height: 1.3; +} + +.diagnostic-kpis { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 0.5rem; + margin-bottom: 1rem; +} + +.diagnostic-kpis > div { + padding: 0.8rem; + border-radius: var(--radius-sm); + background: var(--gray-100); +} + +.diagnostic-kpis span, +.diagnostic-kpis strong { + display: block; +} + +.diagnostic-kpis span { + color: var(--ink-500); + font-size: 0.65rem; +} + +.diagnostic-kpis strong { + margin-top: 0.2rem; + font-family: var(--font-display); + font-size: 1.45rem; + font-weight: 500; +} + +.limitation-list li { + border-bottom: 1px solid var(--gray-100); + color: var(--ink-650); + font-size: 0.82rem; +} + +.limitation-list li::before { + content: "—"; + color: var(--amber-700); +} + +.privacy-panel { + display: grid; + grid-template-columns: 0.7fr 1.3fr; + gap: 1.5rem 2.5rem; + align-items: center; + margin-top: 1rem; + padding: clamp(1.3rem, 3vw, 2rem); + border-radius: var(--radius-lg); + color: var(--white); + background: var(--ink-950); +} + +.privacy-panel .eyebrow { + color: #9bd4d1; +} + +.privacy-panel > p { + grid-column: 1 / -1; + margin: 0; + color: #cad7d5; + font-size: 0.78rem; +} + +.privacy-flow { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.6rem; + text-align: center; +} + +.privacy-flow span { + padding: 0.7rem; + border: 1px solid #536c69; + border-radius: var(--radius-sm); + font-size: 0.75rem; +} + +.privacy-flow small { + color: #b6c5c3; + font-size: 0.6rem; +} + +.privacy-flow__private { + border-color: #c7867d !important; +} + +.privacy-flow__public { + border-color: #72aaa6 !important; +} + +.site-footer { + border-top: 1px solid var(--sand-200); + background: var(--paper-warm); +} + +.site-footer__inner { + display: flex; + align-items: center; + justify-content: space-between; + gap: 2rem; + min-height: 125px; + color: var(--ink-650); + font-size: 0.76rem; +} + +.site-footer strong { + color: var(--ink-950); + font-family: var(--font-display); + font-size: 1rem; +} + +.site-footer p { + max-width: 62ch; + margin: 0.25rem 0 0; +} + +.noscript { + position: fixed; + z-index: 100; + right: 1rem; + bottom: 1rem; + left: 1rem; + padding: 1rem; + border: 2px solid var(--red-700); + border-radius: var(--radius-sm); + background: var(--red-100); + text-align: center; +} + +@media (max-width: 900px) { + .kpi-grid, + .method-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .dashboard-grid--wide, + .explorer-layout, + .split-section, + .privacy-panel { + grid-template-columns: 1fr; + } + + .filter-panel { + position: static; + } + + .cohort-cards { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .privacy-panel > p { + grid-column: auto; + } +} + +@media (max-width: 660px) { + .site-header__inner { + min-height: 68px; + } + + .header-status, + .brand__tagline { + display: none; + } + + .brand__mark { + width: 37px; + height: 37px; + } + + .brand__name { + font-size: 1.05rem; + } + + .primary-nav a { + padding-inline: 0.75rem; + } + + .page-heading--split { + display: block; + } + + .data-stamp { + width: 100%; + margin-top: 1.3rem; + } + + .kpi-grid, + .dashboard-grid, + .method-grid, + .cohort-cards, + .form-grid, + .method-preview { + grid-template-columns: 1fr; + } + + .kpi-card { + min-height: 130px; + } + + .panel__header, + .result-toolbar, + .site-footer__inner { + align-items: flex-start; + flex-direction: column; + } + + .chart-legend { + justify-content: flex-start; + } + + .locked-panel { + display: block; + } + + .locked-panel__icon { + margin-bottom: 1rem; + } + + .privacy-flow { + align-items: stretch; + flex-direction: column; + } + + .diagnostic-kpis { + grid-template-columns: 1fr; + } +} + +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } + + *, + *::before, + *::after { + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + } +} + +@media (forced-colors: active) { + .status-dot, + .legend-swatch, + .risk-bar span, + .timeline li::before { + forced-color-adjust: none; + } +} diff --git a/dashboard/tests/contract.test.mjs b/dashboard/tests/contract.test.mjs new file mode 100644 index 0000000..6c0f096 --- /dev/null +++ b/dashboard/tests/contract.test.mjs @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { createHash, webcrypto } from "node:crypto"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + DataContractError, + REQUIRED_ASSETS, + SCHEMA_VERSION, + loadDashboardData, + validateAssetSet, +} from "../js/data.js"; + +const DASHBOARD_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const PUBLIC_DATA = path.join(DASHBOARD_ROOT, "public", "data"); + +function json(filename) { + return JSON.parse(readFileSync(path.join(PUBLIC_DATA, filename), "utf8")); +} + +function assetSet() { + return Object.fromEntries( + Object.entries(REQUIRED_ASSETS).map(([name, filename]) => [name, json(filename)]), + ); +} + +function publicDataFetch(overrides = new Map()) { + return async (requestUrl) => { + const filename = path.basename(new URL(String(requestUrl)).pathname); + const source = path.join(PUBLIC_DATA, filename); + let bytes; + try { + bytes = overrides.has(filename) ? overrides.get(filename) : readFileSync(source); + } catch { + return { ok: false, async arrayBuffer() { return new ArrayBuffer(0); } }; + } + return { + ok: true, + async arrayBuffer() { + return Uint8Array.from(bytes).buffer; + }, + }; + }; +} + +function filesRecursively(directory) { + return readdirSync(directory).flatMap((name) => { + const target = path.join(directory, name); + return statSync(target).isDirectory() ? filesRecursively(target) : [target]; + }); +} + +test("static shell exposes four semantic navigable views", () => { + const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8"); + assert.match(html, /]*aria-label="Dashboard views"/); + assert.match(html, /One-time holdout<\/strong>Already evaluated<\/small>/); + assert.doesNotMatch(html, /Locked test<\/strong>|Final evaluation<\/small>/); +}); + +test("estimator remains disabled and contains no identifying input", () => { + const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8"); + const estimator = html.match(/
/)?.[0]; + assert.ok(estimator, "estimator section is present"); + assert.match(estimator, /
/); + const controlAttributes = [...estimator.matchAll(/<(?:input|select|textarea)\b([^>]*)>/g)].map( + (match) => match[1], + ); + for (const attributes of controlAttributes) { + assert.doesNotMatch( + attributes, + /(?:name|id)\s*=\s*["'][^"']*(?:vin|plate|address|station|free.?text|zip)[^"']*["']/i, + ); + } +}); + +test("all required generated assets satisfy the browser contract", () => { + const validated = validateAssetSet(assetSet()); + assert.equal(validated.manifest.schema_version, SCHEMA_VERSION); + assert.equal(validated.manifest.development_preview, true); + assert.equal(validated.manifest.population_estimate_allowed, false); + assert.equal(validated.manifest.definitions.locked_test_metrics_published, false); + assert.match(validated.manifest.release_id, /^[0-9a-f]{64}$/); + assert.ok(validated.manifest.model_versions.length > 0); + assert.deepEqual( + validated.filters.partitions, + ["train", "tune", "calibrate"], + ); + assert.ok(validated.overview.rows.length > 0); +}); + +test("browser loader verifies every raw asset digest before rendering", async () => { + const validated = await loadDashboardData({ + basePath: "http://dashboard.test/public/data/", + fetchImplementation: publicDataFetch(), + cryptoImplementation: webcrypto, + }); + assert.equal(validated.manifest.schema_version, SCHEMA_VERSION); + + const changedOverview = Buffer.concat([ + readFileSync(path.join(PUBLIC_DATA, REQUIRED_ASSETS.overview)), + Buffer.from("\n"), + ]); + await assert.rejects( + loadDashboardData({ + basePath: "http://dashboard.test/public/data/", + fetchImplementation: publicDataFetch( + new Map([[REQUIRED_ASSETS.overview, changedOverview]]), + ), + cryptoImplementation: webcrypto, + }), + (error) => + error instanceof DataContractError && + /Integrity verification failed for overview_period_county\.json/.test(error.message), + ); +}); + +test("sha256 manifest covers and matches every approved data asset", () => { + const manifest = json(REQUIRED_ASSETS.shaManifest); + const expectedNames = Object.entries(REQUIRED_ASSETS) + .filter(([name]) => name !== "shaManifest") + .map(([, filename]) => filename) + .sort(); + assert.deepEqual( + manifest.files.map((file) => file.name).sort(), + expectedNames, + ); + for (const entry of manifest.files) { + const digest = createHash("sha256") + .update(readFileSync(path.join(PUBLIC_DATA, entry.name))) + .digest("hex"); + assert.equal(digest, entry.sha256, `${entry.name} digest`); + } +}); + +test("contract fails closed for missing, inconsistent, or sensitive data", () => { + const missing = assetSet(); + delete missing.overview; + assert.throws(() => validateAssetSet(missing), DataContractError); + + const inconsistent = assetSet(); + inconsistent.ageRisk.population_estimate_allowed = true; + assert.throws(() => validateAssetSet(inconsistent), DataContractError); + + const sensitive = assetSet(); + sensitive.scorecards.rows[0].vehicle_token = "not-public"; + assert.throws(() => validateAssetSet(sensitive), DataContractError); + + const extraRowField = assetSet(); + extraRowField.overview.rows[0].note = "unapproved"; + assert.throws(() => validateAssetSet(extraRowField), DataContractError); + + const lockedMetrics = assetSet(); + lockedMetrics.manifest.definitions.locked_test_metrics_published = true; + assert.throws(() => validateAssetSet(lockedMetrics), DataContractError); + + const extraPartition = assetSet(); + extraPartition.filters.partitions.push("locked_test"); + assert.throws(() => validateAssetSet(extraPartition), DataContractError); + + const unsortedVersions = assetSet(); + unsortedVersions.manifest.model_versions = ["z_v1", "a_v1"]; + assert.throws(() => validateAssetSet(unsortedVersions), DataContractError); + + const belowSuppression = assetSet(); + belowSuppression.overview.rows[0].support_rounded = + belowSuppression.manifest.definitions.suppression_min_support - 1; + assert.throws(() => validateAssetSet(belowSuppression), DataContractError); + + const incorrectlyRounded = assetSet(); + incorrectlyRounded.coverage.rows[0].support_rounded += 1; + assert.throws(() => validateAssetSet(incorrectlyRounded), DataContractError); +}); + +test("dashboard source has no external runtime dependency or credential marker", () => { + const sourceFiles = filesRecursively(DASHBOARD_ROOT).filter( + (filename) => + !filename.includes(`${path.sep}public${path.sep}data${path.sep}`) && + !filename.includes(`${path.sep}tests${path.sep}`) && + /\.(?:html|css|js|mjs)$/.test(filename), + ); + for (const filename of sourceFiles) { + const source = readFileSync(filename, "utf8"); + assert.doesNotMatch(source, /(?:postgres(?:ql)?:\/\/|PGPASSWORD|PGHOST|countydata\.)/i); + assert.doesNotMatch(source, /]+src=["']https?:|@import\s+url\(["']?https?:/i); + } +}); diff --git a/docs/dashboard_spec.md b/docs/dashboard_spec.md new file mode 100644 index 0000000..86328e1 --- /dev/null +++ b/docs/dashboard_spec.md @@ -0,0 +1,97 @@ +# Bolt dashboard specification + +## Product language + +Use **Utah Vehicle Health** as the brand, but call the modeled quantity +**next-episode non-pass risk**. The estimator is a cohort estimate, not a +diagnosis, certification, or guarantee. + +## Four-page MVP + +The sections below describe the full product target. The checked-in +development preview intentionally implements a narrower safe subset: binary +pass/non-pass aggregates, supported make/model scorecards, age bands, coverage +quality, and pre-2025 model diagnostics. Four-class charts, uncertainty +intervals, adjusted scorecards, and the prediction lookup remain disabled until +their own reviewed aggregate assets exist. + +### Overview + +- Eligible inspections, pass rate, non-pass rate, and covered-period KPIs +- Quarterly pass/fail/reject/abort trend with blanks shown separately +- Covered-county map; unavailable counties remain gray +- Non-pass risk versus vehicle age with intervals +- Clear notices for partial periods and limited feed coverage + +### Reliability explorer + +- Search supported canonical make/model cohorts +- Compare up to three cohorts across vehicle-age bands +- Observed versus model-adjusted risk toggle +- Outcome-mix bar and uncertainty-aware ranked dot plot +- County, make/model, age band, fuel, program, and period filters +- Support size and interval displayed for every estimate + +### Next-test risk estimator + +Inputs are coarsened, non-identifying attributes: county, supported make/model, +vehicle-age band, fuel, prior episode outcome, time-since-prior band, season, +and approved program category. + +Output a calibrated non-pass probability, uncertainty interval, relevant +baseline, and aggregate factor contributions. Never request VIN, plate, exact +address, station, free text, or current-test diagnostics. + +### Data and methods + +- Coverage timeline and source/year missingness heatmap +- Episode and target definitions +- `slc`/`slco` source-era explanation +- Temporal split, PR-AUC, Brier score, and calibration plot +- Subgroup/source-era performance +- Leakage controls, DMV gaps, partial periods, and limitations + +## Public data contract + +| Dataset | Safe grain | +| --- | --- | +| `data_manifest` | Data cutoff, deterministic release ID, model versions, definitions and exclusions | +| `overview_period_county` | Quarter/year × public county with rounded support and outcome rates | +| `cohort_scorecard` | Approved make/model × age band, optionally coarsened county/fuel | +| `age_risk_curve` | Approved cohort × age point/band with risk, interval and support | +| `prediction_lookup` | Only supported coarsened input combinations and calibrated outputs | +| `filter_catalog` | Publishable categories and valid combinations | +| `model_diagnostics` | Approved partition-level metrics; locked metrics require a separate release gate | +| `coverage_quality` | Source era × year volume, blank rate, linkage and availability | + +Use purpose-built outputs rather than a single high-dimensional browser cube. +Bolt receives only these sanitized, versioned assets—never countydata +credentials or private analytical rows. + +## Publication controls + +- Suppress cells below 100 eligible inspections. +- Also require at least 100 distinct private vehicle tokens in every published + cell; tokens and distinct counts never enter the public asset. +- Suppress when an outcome or its complement has fewer than 10 records. +- Require at least 10 distinct vehicles contributing each binary class. +- Apply complementary suppression so totals cannot reconstruct hidden cells. +- Combine rare categories, coarsen model years, and round displayed counts. +- Recheck thresholds after every filter combination. +- Do not include suppressed rows in browser bundles, API responses, downloads, + analytics logs, or hidden chart layers. +- Downloads contain only the sanitized summary currently displayed. + +## Visual direction + +Use a restrained Utah/desert palette: teal pass, red fail, amber reject, purple +abort, and gray missing. Use probability bars, calibrated dot plots, confidence +bands, and cohort comparisons instead of gauges. Do not rely on color alone. + +## Stretch pages + +- Failure-to-pass journeys with funnels, attempts-to-pass and survival curves +- Four-class outcome probabilities +- Source-scoped OBD early-warning analysis +- Automated aggregate refresh with a dedicated read-only role +- Model-drift monitoring diff --git a/docs/data_inventory.md b/docs/data_inventory.md new file mode 100644 index 0000000..063db46 --- /dev/null +++ b/docs/data_inventory.md @@ -0,0 +1,102 @@ +# Countydata inventory + +Inventory date: 2026-07-15. All inspection was performed with read-only +transactions and aggregate queries; no identifiers were exported. + +## Accessible databases + +- `countydata`: the useful analytical database, approximately 93.4 GB. +- `postgres`: empty local/staging copies of the core vehicle table structure + plus foreign-table links. It is not the analytical source. +- `vattp`: a tiny, unrelated course/event registration and survey application. + It is not suitable for this project and its application records should remain + out of scope. + +## DMV logical dataset + +The following are one logical dataset split into projections joined one-to-one +by `id`; they are not four independent populations. + +| Relation | Approximate role | Key analytical fields | +| --- | --- | --- | +| `data.dmv_tax_json` | Raw JSON and ingest metadata | Full DMV record, file/source metadata | +| `data.dmv_tax` | Core lookup | VIN, registration date, county | +| `data.dmv_tax_search` | Search projection | VIN, registration date, county | +| `data.dmv_tax_vehicle` | Vehicle projection | Make, model, model year, fuel, registration type/place, temporary flag, expiration/emission dates, ZIPs | + +Exact current logical row count: **18,009,278**. + +- Date bounds are 2011-09-07 through 2024-03-13, but the few pre-2016 rows are + outliers and there are only 117 records in 2021. +- The 29 Utah counties are represented. Salt Lake (35.66%), Utah (17.21%), + Davis (10.29%), and Weber (8.01%) account for most records. +- Canonicalized fuel mix by record is 86.23% gasoline, 7.97% diesel, 2.59% + flexible fuel, 2.06% hybrid, 0.70% electric, and 0.24% plug-in hybrid. +- A vehicle appears repeatedly over time: sampled DMV histories had a median of + five registration records. + +## Inspection logical dataset + +These are likewise projections of one logical inspection dataset joined by +`id`. + +| Relation | Approximate role | Key analytical fields | +| --- | --- | --- | +| `data.inspection_json` | Raw JSON and ingest metadata | Full inspection, detailed OBD/readiness/visual fields where supplied | +| `data.inspection` | Core lookup | VIN, test timestamp, ingest timestamp | +| `data.inspection_search` | Outcome/search projection | County/source, overall and OBD results, test/program type | +| `data.inspection_vehicle` | Vehicle projection | Make, model, year, calibration/certificate, station | +| `data.inspection_obd` | OBD summary | Result-reason code, DTC count, permanent-DTC flag | +| `data.inspection_plate` | Identifier lookup | VIN, plate, test timestamp; sensitive and unnecessary for analytics | + +Exact current logical row count: **19,357,287**. + +- Four dates in 1990 are outliers. Normal coverage begins in 2010 and continues + through 2026-06-22; 2026 is partial. +- Real source/county labels are `slc`, `slco`, `utah`, `weber`, `davis`, and + `cache`. The two Salt Lake labels represent different source eras and should + not be blindly treated as different counties. +- Overall results are 71.06% pass, 3.61% fail, 3.56% reject, 2.31% abort, and + 19.45% blank/null or other near-blank values in the raw `overall_result` + field. Most missing overall results belong to the older Utah County feed; its + audited OBD/OBD rows carry a separate result that is usable only as a + provenance-tagged binary pass-versus-non-pass proxy. +- About 89.5% of records use the OBD program and about 9.2% use TSI. +- DTC count is zero in about 90.4% of records and null in about 1.4%; the + remaining values are class-imbalanced and include rare data-quality outliers. +- Raw JSON can contain odometer, vehicle fuel/type/GVWR/cylinders/engine, + transmission, DTCs, PIDs, MIL/readiness status, communication protocol, and + visual inspection fields. These richer fields are concentrated in the newer + `slco`, `davis`, and `cache` feeds rather than statewide history. + +## Longitudinal linkage + +VIN is indexed in both logical datasets and makes longitudinal analysis +possible, but must never appear in public outputs. + +- 88.2% of 10,000 sampled distinct inspection VINs had at least one DMV match. +- The sampled inspection history median was eight visits per vehicle. +- 74.2% of 10,000 sampled distinct DMV VINs had at least one inspection match. +- Extreme repeat counts exist and require invalid/shared-identifier filtering. + +Use a salted one-way internal token if a stable identifier is needed during +feature engineering. Never send VINs or plates to the browser or Bolt. + +## Operational and sensitive relations + +The `imreports` and `fdw_data` schemas include users, invitations, sessions, +event/upload logs, file contents, client network metadata, and remote ingest +state. They are operational rather than analytical and should be excluded. +Foreign tables in `fdw_countydata` mirror source data and are unnecessary when +the normalized `data` tables are available. + +## Primary quality risks + +- Missing years and partial periods can masquerade as real trends. +- Make/model and fuel categories need canonicalization. +- Outcome blanks must not be treated as passes. +- County labels also encode source-system changes and therefore potential drift. +- Rich JSON features are missing by design in older feeds, not missing at + random. +- Same-test OBD/result fields cause target leakage in pre-test prediction. +- Direct identifiers and small groups require aggregation and suppression. diff --git a/docs/development_results.md b/docs/development_results.md new file mode 100644 index 0000000..7a8a6aa --- /dev/null +++ b/docs/development_results.md @@ -0,0 +1,136 @@ +# Development run results + +Run date: 2026-07-15 + +These results validate the engineering and modeling pipeline. They are **not +population estimates**: the development extract samples inspection-table pages +and then retrieves complete histories for up to 10,000 vehicles, which +over-represents vehicles with more inspection records. + +## Label audit + +The raw Utah County feed leaves `overall_result` blank for 3,747,862 records. +An aggregate-only server audit found 3,469,102 Utah `obd`/`OBD` records with a +controlled pass, fail, reject, or abort value in `obd_result`. Across non-Utah +feeds where both fields were recognized, the two fields agreed on binary pass +versus non-pass 99.30% of the time, but their fail-versus-reject categories were +not interchangeable. + +Label contract v3 therefore permits the Utah OBD value only as a provenance- +tagged binary proxy. TSI, `other/C`, `B`, blank, and unknown values stay +unlabeled. Four-class analysis must use `overall_result` exclusively. This also +matches the official program distinction between a readiness rejection and a +failed inspection; see the [Utah inspection requirements](https://dmv.utah.gov/register/inspections/) +and [program definition of rejection](https://www.utah.gov/pmn/files/1155003.pdf). + +In the development extract, the rule restored 11,334 labels and left 1,281 +events unlabeled. The label-source field remains private audit metadata and is +not a model feature. + +## Pipeline reconciliation + +| Stage | Rows | +| --- | ---: | +| Source events written after VIN validation | 83,552 | +| Clean events after duplicate/conflict handling | 80,190 | +| Inspection episodes | 69,588 | +| Eligible returning targets | 44,659 | + +The sample contains 9,996 retained vehicle tokens. The extraction manifest, +compressed-file SHA-256, mart manifest, Parquet SHA-256, and row counts all +reconcile. Every mart invariant reports zero violations. Raw VIN and raw OBD +result are absent from the staging output schema, feature mart, and model +features. + +| Temporal partition | Eligible targets | Never-fit audit targets | +| --- | ---: | ---: | +| Train, 2016–2022 | 31,745 | 3,129 | +| Tune, 2023 | 4,972 | 526 | +| Calibrate, 2024 | 2,633 | 277 | +| Locked test, 2025 | 4,451 | 480 | +| Shadow, 2026 partial | 858 | 106 | + +Audit-bucket vehicles are excluded from fitting, tuning, and calibration. The +normal trainers do not calculate 2025 outcomes or metrics without the explicit +locked-evaluation flag. + +## Baseline results before the locked test + +The table below uses only non-audit vehicles. Average precision is the project's +PR-AUC summary. The calibrated logistic row is shown only on the partition used +to fit the calibrator and is therefore a calibration diagnostic, not an +independent final estimate. + +| Partition | Model | PR-AUC | Brier | ROC-AUC | Precision at top 10% | +| --- | --- | ---: | ---: | ---: | ---: | +| 2023 tune | Training prevalence | 0.125 | 0.1092 | 0.500 | 0.125 | +| 2023 tune | Previous episode, literal | 0.166 | 0.1723 | 0.582 | 0.285 | +| 2023 tune | Logistic | **0.282** | **0.1012** | **0.707** | **0.315** | +| 2024 calibrate | Training prevalence | 0.113 | 0.1002 | 0.500 | 0.113 | +| 2024 calibrate | Previous episode, literal | 0.148 | 0.1677 | 0.577 | 0.253 | +| 2024 calibrate | Logistic | 0.257 | 0.0936 | 0.690 | 0.308 | +| 2024 calibrate | Logistic + Platt | **0.257** | **0.0936** | **0.690** | **0.308** | + +The selected logistic regularization was `C=0.03`. It converged in 2,390 of the +5,000 allowed iterations; Platt calibration converged in five iterations. All +five candidates and all stored probabilities passed explicit convergence and +finite-value checks. + +The literal previous-outcome baseline is useful as a ranking sanity check but +produces overconfident zero/one probabilities, explaining its poor Brier and log +loss. The transparent logistic model is the current development leader: on the +2023 tuning partition it more than doubles prevalence PR-AUC and raises top-10% +precision from 12.5% to 31.5%. + +## One-time development holdout + +The nonlinear live-verification command invoked the explicit 2025 gate after +its feature contract and four-candidate grid had already been fixed and its +candidate had been selected only on 2023. No test-driven model change was made. +At that point the development specification was frozen and the already-fixed +logistic model was evaluated once for a direct comparison. These are +development-sample holdout results, not final population claims. + +| 2025 non-audit model | PR-AUC | Brier | ROC-AUC | Precision at top 10% | +| --- | ---: | ---: | ---: | ---: | +| Training prevalence | 0.123 | 0.1076 | 0.500 | 0.123 | +| Previous episode, literal | 0.158 | 0.1731 | 0.576 | 0.268 | +| Logistic + Platt | **0.261** | **0.1011** | **0.693** | **0.312** | +| Histogram gradient boosting + Platt | 0.238 | 0.1024 | 0.690 | 0.292 | + +The holdout contains 3,971 non-audit episodes from 3,734 vehicles. The +never-fit audit contains another 480 episodes from 450 vehicles; calibrated +logistic PR-AUC is 0.253 and Brier is 0.0859 there. The logistic model beats +both simple baselines on the project's headline metrics and remains the chosen +development model. The tree did not provide a decisive pre-test improvement +that justified its added complexity, and no further 2025-informed tuning is +permitted. + +## Remaining gates + +1. Run a complete, contiguous bounded extraction for publishable population + aggregates; the page-sampled development cohort cannot support dashboard + prevalence or county rankings. +2. Treat any later full-data 2025 result as confirmatory rather than a pristine + unseen test, because the development sample's holdout has now been opened. +3. Run the frozen subgroup/source-era report and clustered uncertainty + analysis on the complete extraction. +4. Replace the checked-in development-preview bundle with complete-data + aggregates only after the population-publication review passes. + +## Sanitized dashboard preview + +The repository now includes a fail-closed exporter and a static four-view +dashboard shell. The preview exporter reads only train, tune, and calibration +rows dated before 2025. It rejects any model manifest that says the holdout was +evaluated, rejects any metric row outside the three approved partitions, and +marks every asset `development_preview=true` and +`population_estimate_allowed=false`. + +Published cells must clear minimum episode and distinct-vehicle thresholds, +including both binary-class complements. Supports are rounded, direct and +pseudonymous identifiers stay private, and a checksum manifest covers the +approved JSON bundle. Private model manifests bind the exact metrics files, and +the public manifest exposes a deterministic release ID plus model versions for +provenance. The site disables the estimator because no privacy-reviewed +prediction lookup exists. diff --git a/docs/model_card.md b/docs/model_card.md new file mode 100644 index 0000000..fe11c9b --- /dev/null +++ b/docs/model_card.md @@ -0,0 +1,197 @@ +# Utah Vehicle Health model card + +Last updated: 2026-07-15 + +Status: **development prototype; not approved for production or population claims** + +## Model summary + +Utah Vehicle Health estimates the probability that the first attempt of a +returning vehicle's next emissions-inspection episode will be a non-pass. The +current selected model is a regularized logistic regression followed by Platt +probability calibration. It is an inspection-history model, not a mechanical +health, safety, roadworthiness, or legal-compliance model. + +This model card summarizes the current page-sampled development run. The full +research and product contracts are in the [project charter](project_charter.md), +[modeling protocol](modeling_protocol.md), and +[dashboard specification](dashboard_spec.md). + +## Intended use and prohibited use + +Intended uses are to validate the data-engineering and modeling pipeline, +compare leakage-safe model candidates, study aggregate patterns, and support a +clearly marked development dashboard preview. + +Do not use the model to: + +- make decisions about an individual vehicle, owner, registration, inspection, + or station; +- diagnose a vehicle, guarantee an inspection result, or infer safety or + roadworthiness; +- rank or penalize people, counties, programs, or inspection stations; +- make causal claims from observed associations; or +- report statewide prevalence, county rankings, or production performance from + the current development sample. + +The public estimator remains disabled because no privacy-reviewed prediction +lookup has been approved. + +## Prediction unit and target + +The prediction is made immediately before a new inspection episode begins. An +episode groups consecutive attempts no more than 30 days apart, and only its +first attempt is the supervised target. A vehicle must have at least one prior +completed episode to enter the returning-vehicle cohort. + +The binary target is: + +- `0`: recognized pass; +- `1`: recognized fail, reject, or abort; and +- unlabeled: blank, null, or unrecognized results. + +A recognized overall result is preferred. For the older Utah County OBD feed, +a controlled OBD result may fill a blank overall result only under the narrow +source/program/test contract documented in the +[modeling protocol](modeling_protocol.md#source-specific-label-contract). That +proxy is approved only for binary pass versus non-pass; it must not support a +four-class interpretation. Label provenance is private audit metadata and is +not a predictor. + +## Development data and representativeness + +The current extract samples inspection-table pages and then retrieves complete +histories for up to 10,000 vehicles. This over-represents vehicles with more +inspection records. The run contains 83,552 retained source events, 69,588 +episodes, and 44,659 eligible returning targets, but those counts do not make +the sample population-representative. + +Coverage is limited to participating county/source feeds and changes over time. +It does not represent all 29 Utah counties. Source-system transitions, missing +years, and partial periods can resemble real changes in risk. See the +[data inventory](data_inventory.md) and +[development results](development_results.md) for the audited scope. + +## Features and leakage exclusions + +The selected model uses information available before the target episode: + +- vehicle age; +- prior episode and attempt counts; +- days since the prior episode and prior adverse outcome; +- prior non-pass rate; +- prior first and final outcomes; +- previously observed make and model; +- public county; and +- target season. + +Categorical mappings and all preprocessing are fit on training data only. The +current model does not yet include DMV enrichment or rich same-test OBD fields. + +Excluded inputs include direct or pseudonymous identifiers, station +information, the target attempt's result or diagnostics, later attempts in the +target episode, future records, full-history aggregates, label provenance, and +preprocessing learned from evaluation periods. The complete exclusion contract +is in [modeling_protocol.md](modeling_protocol.md#leakage-exclusions). + +## Chronology and holdout status + +The fixed temporal design is: + +| Period | Role | +| --- | --- | +| 2010-2015 | Historical context only | +| 2016-2022 | Fit preprocessing and models | +| 2023 | Select hyperparameters and compare candidates | +| 2024 | Fit probability calibration and inspect calibration behavior | +| 2025 | One-time development holdout | +| 2026 partial | Shadow monitoring only | + +During live verification, the explicit 2025 gate was opened once after both +candidate specifications and the tree search grid had already been fixed from +pre-2025 data. No model was changed in response. The already-fixed logistic +model was then evaluated once for a direct comparison. Consequently, this +page-sampled 2025 cohort is no longer a pristine unseen test, and no further +2025-informed tuning is permitted. Any complete-data 2025 analysis must be +described as confirmatory. The detailed audit trail is in +[development_results.md](development_results.md#one-time-development-holdout). + +## Candidate comparison and selected model + +The candidates were training prevalence, the literal previous-episode outcome, +regularized logistic regression, and histogram gradient boosting. Results below +use non-audit development rows. + +| Evaluation | Model | PR-AUC | Brier | ROC-AUC | Top-10% precision | +| --- | --- | ---: | ---: | ---: | ---: | +| 2023 tuning | Training prevalence | 0.125 | 0.1092 | 0.500 | 0.125 | +| 2023 tuning | Previous episode | 0.166 | 0.1723 | 0.582 | 0.285 | +| 2023 tuning | Logistic | **0.282** | **0.1012** | **0.707** | **0.315** | +| 2025 one-time holdout | Logistic + Platt | **0.261** | **0.1011** | **0.693** | **0.312** | +| 2025 one-time holdout | Gradient boosting + Platt | 0.238 | 0.1024 | 0.690 | 0.292 | + +The selected logistic model uses `C=0.03`. It satisfied explicit convergence +and finite-value checks. Platt scaling was fit on the 2024 calibration cohort; +on that same cohort, PR-AUC was 0.257 and Brier score was 0.0936. Those 2024 +values are calibration diagnostics, not independent final performance. + +The logistic model remains selected because it beat both simple baselines and +the more complex tree on the one-time development holdout while remaining more +transparent. The separate never-fit vehicle audit also supported the logistic +model, but the development sample is too limited for final generalization or +fairness claims. + +## Coverage, fairness, and privacy limitations + +- The model applies only to returning vehicles with recognizable labels and + sufficient prior history; cold-start behavior is not established. +- Geography and source era are entangled. Performance may shift when a feed, + county program, vehicle mix, or label process changes. +- Reject and abort outcomes can reflect readiness or process issues rather than + mechanical failure. +- No protected attributes are modeled, but their absence does not establish + fairness. Subgroup sample sizes, errors, and calibration still require review. +- Make/model normalization and complete-data subgroup analysis are unfinished. +- The current probability calibration has not yet received independent, + vehicle-clustered uncertainty analysis or external validation. + +Private analytical artifacts remain local. Public assets contain only reviewed, +rounded aggregates that satisfy episode, distinct-vehicle, and binary-class +suppression thresholds. Direct identifiers, private linkage values, raw rows, +and row-level predictions are outside the public contract. See +[dashboard_spec.md](dashboard_spec.md#publication-controls). + +## Monitoring + +Before any deployment, monitoring must cover source-level volume, label +recognition, outcome prevalence, source/program mix, missingness, unseen +categories, score distributions, PR-AUC, Brier score, and calibration. Reviews +must explicitly separate the Salt Lake source transition, newer rich feeds, DMV +coverage gaps, and the partial 2026 period. Alert thresholds and a response plan +remain to be defined. + +## Reproducibility + +The pipeline uses fixed temporal boundaries and a fixed random seed. Python +dependencies are pinned in [requirements.txt](../requirements.txt). Private +manifests record input lineage, software/model configuration, convergence, row +reconciliation, and checksums without publishing private paths or values. SQL +transforms and validation queries are versioned under [sql](../sql), model +runners are under [scripts](../scripts), and automated checks are under +[tests](../tests). Normal training commands leave the 2025 evaluation gate +closed unless an explicit flag is supplied. + +## Remaining approval gates + +1. Run a complete, contiguous, bounded extraction and rebuild the frozen + pipeline without page-sampling bias. +2. Treat complete-data 2025 results as confirmation and reserve a genuinely new + period or external dataset for future unseen evaluation. +3. Complete subgroup/source-era reporting, vehicle-clustered uncertainty, + calibration diagnostics, and fairness review. +4. Finish make/model normalization, episode-gap sensitivity checks, and the + preregistered inspection-only versus DMV-enhanced ablation. +5. Pass privacy and publication review before replacing the development-preview + aggregates or enabling any prediction lookup. +6. Define monitoring thresholds, ownership, rollback criteria, and a model + update policy before operational use. diff --git a/docs/modeling_protocol.md b/docs/modeling_protocol.md new file mode 100644 index 0000000..ee06b28 --- /dev/null +++ b/docs/modeling_protocol.md @@ -0,0 +1,141 @@ +# Leakage-safe modeling protocol + +## Prediction unit + +Score the **first attempt of the next inspection episode** immediately before +check-in. Consecutive attempts for the same private vehicle token belong to the +same episode when they are no more than 30 days apart. Re-run the analysis with +14- and 45-day gaps as sensitivity checks. + +This prevents rapid fail/retest sequences from dominating the target. Those +within-episode attempts belong in the separate failure-to-pass journey analysis. + +## Eligibility + +- Use 2010-2015 events only as historical context; supervised targets begin in + 2016. +- Exclude the four 1990 date outliers. +- Require a recognized first-attempt outcome and at least one prior completed + episode for the returning-vehicle model. +- Keep first-observed vehicles as a separate cold-start cohort. +- Normalize `slc` and `slco` to Salt Lake County for geography while preserving + source era for drift reporting. +- Deduplicate exact uploads and quarantine shared/test/placeholder identifiers + using preregistered rules for impossible conflicts or extreme activity. +- Do not require a DMV match; retain explicit match and staleness indicators. + +Every exclusion must appear in a cohort-flow report. + +## Source-specific label contract + +The normalized target prefers a recognized `overall_result` for every source. +The older Utah County feed is the sole exception: when its overall result is +blank, `obd_result` may supply the binary target only when the source is +`utah`, the program is `obd`, the test type is `OBD`, and the controlled value +is pass, fail, reject, or abort. `B`, blank, TSI, `other/C`, and unknown values +remain unlabeled. Every target retains `target_outcome_label_source` as private +audit metadata, and that field is never a predictor. + +This is a binary pass-versus-non-pass proxy, not a four-class substitution. +Across non-Utah feeds where both fields are recognized, an aggregate audit found +99.30% agreement on pass versus non-pass, while the fail/reject distinction was +not interchangeable. Utah program rules also distinguish a readiness rejection +from a failed inspection. Future four-class analysis must therefore require +`target_outcome_label_source = 'overall_result'`. See the official +[Utah inspection requirements](https://dmv.utah.gov/register/inspections/) and +[program definition of rejection](https://www.utah.gov/pmn/files/1155003.pdf). + +## Point-in-time features + +Every historical window ends strictly before the target episode: + +- prior episode and attempt counts; +- previous episode first/final outcome; +- expanding and trailing prior pass/fail/reject/abort counts and rates; +- days since the previous episode and prior adverse result; +- attempts required in prior episodes; +- vehicle age and canonical make/model from prior information; +- physical county and target month/season; +- latest DMV record dated before the target, registration count/recency, fuel, + and DMV match/staleness indicators; and +- missingness indicators. + +The stable core model is inspection-only. The DMV-enhanced model is an explicit +ablation because DMV history ends in March 2024 and is nearly absent in 2021. + +## Leakage exclusions + +The MVP must not use: + +- the target attempt's overall/OBD result, result reason, DTC count, MIL or + readiness values, PIDs, visual checks, measurements, certificate, or + calibration fields; +- later attempts or eventual outcome from the target episode; +- station or station-level outcome statistics; +- raw VIN, private token, plate, ZIP, or other identifiers as features; +- target-row vehicle attributes when a prior/static source is available; +- future DMV records; +- full-history aggregates; or +- preprocessing, category mappings, target encodings, or imputation learned + from validation/test data. + +Timestamp ties must be resolved before lag/window calculations. Cumulative +windows end at the preceding event or episode. + +## Fixed evaluation timeline + +| Partition | Target dates | Purpose | +| --- | --- | --- | +| Historical context | 2010-2015 | Lag features only | +| Train | 2016-2022 | Fit preprocessing and models | +| Tune | 2023 | Hyperparameters and selection | +| Calibrate | 2024 | Probability calibration and thresholds | +| Locked test | 2025 | Final reported performance | +| Shadow drift | 2026-01-01 to 2026-06-22 | Monitoring only | + +The page-sampled development extract's 2025 gate was opened once during live +verification after both candidate specifications were fixed. It is therefore +a one-time development holdout, not a pristine future test. No model change was +made from it; a later complete-data 2025 run is confirmatory. See +[development_results.md](development_results.md) for the audit trail. The table +continues to define the frozen chronology for a complete extraction. + +Repeated vehicles may cross ordinary time partitions because returning-vehicle +prediction is the deployment scenario. Separately reserve 10% of keyed vehicle +buckets as a never-fit VIN audit and report its 2025 performance as an unseen- +vehicle stress test. Never use random row splitting. + +## Baselines and candidate model + +1. Training prevalence +2. Repeat the previous episode's first outcome +3. Regularized logistic regression with age splines and one-hot categoricals +4. A boosted-tree model +5. Inspection-only versus inspection-plus-DMV ablation + +Keep validation and test sets at natural prevalence. If pass rows are sampled +for training, preserve sampling probabilities and recalibrate on the untouched +2024 partition. + +## Metrics + +Headline binary metrics: + +- non-pass PR-AUC; +- Brier score and log loss; +- calibration intercept, slope, and reliability curve; +- precision, recall, and lift at fixed review capacities; and +- ROC-AUC as secondary context. + +Use vehicle-clustered bootstrap confidence intervals. Report results by county, +source era, vehicle-age band, prior outcome, fuel, history depth, and DMV +match/staleness. Four-class analysis adds class-specific and macro PR-AUC, +multiclass log loss/Brier score, calibration, and a confusion matrix. + +## Drift contract + +Monitor monthly/source-level volume, label recognition, outcome prevalence, +blank rate, source/program mix, missingness, DMV staleness, unseen categories, +prediction distribution, PR-AUC, Brier score, and calibration. Treat the +`slc`→`slco` transition, new rich feeds, DMV 2021 gap, DMV 2024 endpoint, and +partial 2026 period as explicit stress cases rather than ordinary random drift. diff --git a/docs/project_charter.md b/docs/project_charter.md new file mode 100644 index 0000000..870e2a9 --- /dev/null +++ b/docs/project_charter.md @@ -0,0 +1,122 @@ +# Utah Vehicle Health project charter + +## Working title + +**Utah Vehicle Health** + +*What millions of emissions inspections reveal about the next test across the +Utah county feeds available in this dataset.* + +## Product promise + +Explain how vehicle age, type, location, and prior inspection history relate to +the chance of passing the first attempt of the next emissions-inspection +episode. + +The product measures **emissions-inspection outcomes**. It must not describe its +score as a diagnosis of overall mechanical reliability, roadworthiness, safety, +or legal compliance. + +## Primary research question + +> Using only information available before an inspection episode begins, how +> accurately and reliably can we estimate whether its first attempt will pass? + +An episode groups attempts for the same internal vehicle token when the gap +between consecutive attempts is 30 days or less. The target is the first attempt +of a new episode, not every rapid retest. + +## Primary target + +- `0 — pass`: normalized `PASS` or `P` +- `1 — non-pass`: normalized `FAIL`, `F`, `REJECT`, or `ABORT` +- Blank, null, and unrecognized results are unlabeled. They remain in chronology + and coverage reporting but are excluded as supervised targets. + +Because reject and abort can reflect process/readiness problems rather than +mechanical failure, the project will also report: + +- a four-class pass/fail/reject/abort analysis; and +- a fail-versus-pass sensitivity analysis that excludes reject and abort. + +## Scope + +The returning-vehicle MVP uses universal inspection-history fields and prior +DMV information where it is point-in-time valid. It covers participating +inspection county/source feeds, not all 29 Utah counties. + +The rich OBD/odometer fields are a later, source-specific extension for newer +`slco`, `davis`, and `cache` records. They are not part of the statewide-style +historical baseline. + +## Deliverables + +1. A reproducible, read-only extraction and private pseudonymization pipeline. +2. A point-in-time episode/feature mart with an auditable exclusion report. +3. Prevalence, previous-outcome, and logistic-regression baselines. +4. One calibrated boosted-tree model and an inspection-only versus DMV-enhanced + ablation. +5. A locked temporal evaluation with subgroup and source-era diagnostics. +6. Versioned, suppressed public aggregates for a four-page Bolt dashboard. +7. A model card and data/methodology page documenting limitations. + +## Headline success criteria + +- Beat both the training-prevalence and previous-episode-outcome baselines on + 2025 PR-AUC and Brier score. +- Produce calibrated probabilities, not just class labels. +- Report performance by county/source era, vehicle-age band, history depth, and + DMV match/staleness. +- Reproduce all published charts from versioned sanitized outputs. +- Export no VIN, plate, ZIP, station, raw JSON, operational record, or + row-level prediction to Bolt. +- Suppress public cells with fewer than 100 eligible inspections or fewer than + 10 observations in an outcome or its complement, with complementary + suppression where totals could reveal a hidden cell. + +## Initial feasibility result + +Under label contract v3, the fixed-seed aggregate-only query in +`sql/10_episode_cohort_feasibility.sql` produced 8,916 eligible +returning-vehicle episodes from 1,730 of 2,000 sampled vehicle histories. The +first-attempt non-pass rate was 11.80%, and the median gap from the prior +episode was about 372 days. Of those targets, 1,732 use the documented Utah OBD +binary proxy because that source leaves its overall-result field blank. This +supports both the episode definition and a calibrated binary model. The sample +is for pipeline feasibility, not a population estimate. + +The 2025 sample showed a longer median gap and different outcome mix, reinforcing +the need for source-era drift reporting and a locked chronological test. + +## Development holdout status + +During live verification, the explicit 2025 gate was opened once on the +page-sampled development extract after both candidate specifications had been +fixed from pre-2025 data. No test-informed model change was made. The frozen +logistic model remains selected, and a later complete-data 2025 analysis will +be treated as confirmatory rather than described as a pristine unseen test. +The audit trail and results are recorded in +[development_results.md](development_results.md). + +## Non-goals + +- Diagnosing an individual vehicle +- Certifying that a vehicle will pass +- Ranking or accusing inspection stations +- Identifying owners or accepting VIN/plate input +- Making causal claims about county programs +- Treating missing outcomes as passes + +## Staged build + +1. **Foundation:** episode definition, label normalization, exclusions, secure + extraction, and coverage checks. +2. **Baseline:** inspection-history feature mart and transparent baselines. +3. **Enrichment:** canonical vehicle dimension and point-in-time DMV features. +4. **Modeling:** tree model, calibration, locked test, drift and subgroup audits. +5. **Product:** sanitized aggregate export and Bolt MVP. +6. **Stretch:** failure-to-pass journeys, multiclass probabilities, and a + clearly scoped rich-OBD model. + +See [modeling_protocol.md](modeling_protocol.md) and +[dashboard_spec.md](dashboard_spec.md) for the detailed contracts. diff --git a/docs/project_options.md b/docs/project_options.md new file mode 100644 index 0000000..cf3dd18 --- /dev/null +++ b/docs/project_options.md @@ -0,0 +1,161 @@ +# Project options based on the countydata inventory + +Inventory date: 2026-07-15 + +> **Selected:** Option 1, Utah Vehicle Health. The implementation contract is in +> [project_charter.md](project_charter.md). + +## What is actually available + +The useful analytical source is the `countydata` PostgreSQL database, which is +about 93.4 GB. Its normalized tables are projections of two large logical +datasets, so their row counts should not be added together: + +- 18,009,278 DMV registration/tax records with county, registration date, + make, model, model year, fuel type, registration type/place, expiration and + emissions dates, ZIP codes, and temporary-registration status. +- 19,357,287 inspection records with county/source, timestamp, make, model, + model year, station, result, OBD result, test/program type, and OBD summary. +- Raw inspection JSON adds odometer, fuel, engine, transmission, vehicle type, + readiness monitors, visual checks, DTCs, PIDs, and communication protocol for + newer county/source feeds. + +The vehicle histories link well without exposing identifiers: 88.2% of a +10,000-VIN inspection sample had DMV history, and the sampled median was eight +inspection visits per vehicle. In the other direction, 74.2% of sampled DMV +vehicles had an inspection match. + +Important limitations: + +- DMV coverage is strong in 2016-2020 and 2022 through March 2024, but only 117 + rows are dated 2021. Treat 2024 as partial. +- Inspection coverage is strong from 2010 through June 2026, but the four rows + dated 1990 are clear date outliers. +- Raw inspection `overall_result` labels are imbalanced: 71.06% pass, 3.61% + fail, 3.56% reject, 2.31% abort, and 19.45% blank/null. Most missing values + come from the older Utah County feed; its separately encoded OBD result can + support a provenance-tagged binary target, but not the four-class analysis. +- Rich OBD/odometer fields are source-dependent. They are essentially absent + from the older `slc`, `utah`, and `weber` feeds and concentrated in `slco`, + `davis`, and `cache` records from late 2024 onward. +- Categories require cleaning: fuel values differ by case and punctuation, and + inspection make values include aliases such as `TOYOTA`/`TOYOT` and + `CHEVROLET`/`CHEVR`. + +## Ranked ideas + +### 1. Utah Vehicle Health and Reliability Observatory + +Build a model that estimates the chance a vehicle will fail, reject, or abort +its next emissions inspection using only information known before that test: +vehicle age, canonical make/model, county, season, fuel, prior test outcomes, +time since the previous test, and longitudinal DMV history. + +The dashboard could include: + +- make/model/year reliability scorecards with uncertainty intervals; +- risk-versus-age curves and county comparisons; +- prior-failure and repeat-test patterns; +- calibrated individual what-if estimates without accepting or displaying a + VIN; and +- a methodology/data-quality page showing drift and missingness. + +This is the strongest overall option because it combines SQL/data engineering, +entity resolution, longitudinal feature engineering, classification or +survival analysis, model explainability, and a compelling public dashboard. + +Guardrail: do not use same-test OBD result, DTC count, or overall result as +features when claiming to predict an outcome before the test. That would be +target leakage. Use a time-based holdout and report PR-AUC, calibration, and +Brier score rather than accuracy alone. + +### 2. Failure-to-Pass Journey Analyzer + +Follow vehicles forward after a fail or reject and model how many attempts and +how much time it takes to achieve a pass. Compare journeys by vehicle age, +canonical make/model, program, county/source, and failure history using +time-to-event or competing-risk methods. + +The dashboard could use journey funnels, transition diagrams, survival curves, +and a cohort comparison tool. This is an unusually good fit for the repeated +histories and can become a major page within option 1 instead of a separate app. + +### 3. Utah EV and Hybrid Transition Atlas + +Deduplicate repeated registrations into vehicle-by-period snapshots, estimate +electric and hybrid share by county and ZIP, measure transitions in the vehicle +fleet, and forecast adoption scenarios. + +The dashboard could use a choropleth, adoption curves, county rankings, and a +make/model explorer. This is visually strong and easy to explain, but the DMV +2021 gap and March 2024 endpoint make honest uncertainty and partial-period +handling essential. + +### 4. OBD Early-Warning Lab + +Scope the analysis to the richer `slco`, `davis`, and `cache` feeds. Use +odometer, vehicle age, fuel, engine, readiness monitors, communication protocol, +MIL state, and prior history to distinguish likely pass, fail, reject, and +abort outcomes and discover common failure signatures. + +The dashboard could show diagnostic pathways, readiness-monitor patterns, and +failure signatures by vehicle cohort. It is technically novel, but it is not a +statewide study because rich-feature coverage begins mainly in 2024-2025. + +### 5. Risk-Adjusted Inspection Station Consistency + +Fit a hierarchical model of station outcomes after controlling for vehicle age, +make/model, program, county, and time. Use empirical-Bayes shrinkage or control +charts to flag unusual reject, abort, or failure rates and detect process drift. + +This is excellent applied statistics and anomaly detection. Public results +should anonymize station identifiers, suppress small groups, show uncertainty, +and describe anomalies as review signals rather than evidence of misconduct. + +### 6. AutoClean: Government Vehicle Data Entity Resolution + +Use the high VIN match rate as weak supervision to learn canonical make/model +mappings between messy inspection strings and cleaner DMV values. Compare +rules, fuzzy matching, and a supervised ranking model, then quantify how much +normalization improves downstream analytics. + +The dashboard could show before/after category fragmentation, match confidence, +and correction examples using non-identifying values. This is a particularly +strong data-engineering project and could also serve as the first pipeline stage +for option 1. + +### 7. Inspection Demand Forecast and Operations Dashboard + +Forecast daily or weekly test volume by county and station using timestamps, +seasonality, holidays, long-term trend, and recent history. Add change-point and +volume-anomaly detection. + +The dashboard could show workload forecasts, day/hour heatmaps, forecast +intervals, and historical disruptions. It is feasible and useful, although it +has less machine-learning depth than the reliability project unless forecasting +and anomaly evaluation are developed carefully. + +### 8. Vehicle Longevity and Survival Index + +Treat the repeated DMV and inspection histories as censored longitudinal data. +Estimate how long makes, models, fuel types, and model-year cohorts remain +active, using Kaplan-Meier curves and a Cox or gradient-boosted survival model. + +The dashboard could answer "Which vehicles stay on Utah roads the longest?" +The main methodological challenge is that disappearance from the data can mean +sale, relocation, incomplete coverage, or retirement, so the result must be +described as observed-system retention rather than mechanical lifespan. + +## Recommended project shape + +Use option 1 as the main story and option 6 as its data-engineering foundation: + +1. Build an incremental, read-only SQL extraction and canonical vehicle model. +2. Create one row per eligible upcoming inspection using only prior information. +3. Compare a transparent logistic baseline with a tree model. +4. Validate on a later time period and separately by county/source. +5. Export only aggregate scorecards, curves, and de-identified model outputs. +6. Build the public Bolt dashboard over those safe outputs. + +This gives the project a coherent end-to-end narrative across data engineering, +analytics, machine learning, responsible validation, and product design. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..077aad3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +duckdb==1.4.5 +joblib==1.5.3 +numpy==2.0.2 +psycopg2-binary==2.9.12 +scipy==1.13.1 +scikit-learn==1.6.1 +threadpoolctl==3.6.0 diff --git a/scripts/build_feature_mart.py b/scripts/build_feature_mart.py new file mode 100644 index 0000000..124c364 --- /dev/null +++ b/scripts/build_feature_mart.py @@ -0,0 +1,727 @@ +"""Build a private, point-in-time inspection feature mart with DuckDB. + +Inputs are either a contiguous set of bounded monthly CSV.gz exports or the +single complete-history development sample. Every input must have the manifest +written by the corresponding secure exporter. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import json +import os +import re +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +import duckdb + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +PRIVATE_DATA_ROOT = (PROJECT_ROOT / "data/private").resolve() +DEFAULT_INPUT = PRIVATE_DATA_ROOT / "inspection_batches" +DEFAULT_DATABASE = PRIVATE_DATA_ROOT / "warehouse/vehicle_health.duckdb" +DEFAULT_OUTPUT = PRIVATE_DATA_ROOT / "marts/inspection_feature_mart.parquet" +SQL_ROOT = PROJECT_ROOT / "sql/local" + +EXPECTED_COLUMNS = ( + "internal_event_id", + "vehicle_token", + "vehicle_bucket", + "event_ts", + "source_era", + "public_county", + "canonical_outcome", + "outcome_label_source", + "program_type", + "test_type", + "observed_make", + "observed_model", + "observed_model_year", +) +BOUNDED_KIND = "bounded_batch" +DEVELOPMENT_KIND = "vehicle_history_development_sample" +HEX_64 = re.compile(r"^[0-9a-f]{64}$") +FINGERPRINT = re.compile(r"^[0-9a-f]{16,64}$") +MEMORY_LIMIT = re.compile(r"^[1-9][0-9]*(?:MB|GB|TB)$", re.IGNORECASE) + +SQL_FILES = ( + "20_stage_events.sql", + "21_build_episodes.sql", + "22_build_features.sql", + "23_validate_mart.sql", +) + +CREATE_STAGING_SQL = """ +CREATE TABLE stg_events ( + batch_file VARCHAR NOT NULL, + batch_kind VARCHAR NOT NULL, + batch_start TIMESTAMP NOT NULL, + batch_end TIMESTAMP NOT NULL, + internal_event_id VARCHAR, + vehicle_token VARCHAR, + vehicle_bucket VARCHAR, + event_ts VARCHAR, + source_era VARCHAR, + public_county VARCHAR, + canonical_outcome VARCHAR, + outcome_label_source VARCHAR, + program_type VARCHAR, + test_type VARCHAR, + observed_make VARCHAR, + observed_model VARCHAR, + observed_model_year VARCHAR +) +""" + +# The dialect and every input column are explicit. In particular, do not let a +# sample without quoted values convince DuckDB that quote handling is disabled. +INGEST_SQL = """ +INSERT INTO stg_events +SELECT + ? AS batch_file, + ? AS batch_kind, + ?::TIMESTAMP AS batch_start, + ?::TIMESTAMP AS batch_end, + source.internal_event_id, + source.vehicle_token, + source.vehicle_bucket, + source.event_ts, + source.source_era, + source.public_county, + source.canonical_outcome, + source.outcome_label_source, + source.program_type, + source.test_type, + source.observed_make, + source.observed_model, + source.observed_model_year +FROM read_csv( + ?, + header = true, + auto_detect = false, + delim = ',', + quote = '"', + escape = '"', + nullstr = '', + strict_mode = true, + columns = { + 'internal_event_id': 'VARCHAR', + 'vehicle_token': 'VARCHAR', + 'vehicle_bucket': 'VARCHAR', + 'event_ts': 'VARCHAR', + 'source_era': 'VARCHAR', + 'public_county': 'VARCHAR', + 'canonical_outcome': 'VARCHAR', + 'outcome_label_source': 'VARCHAR', + 'program_type': 'VARCHAR', + 'test_type': 'VARCHAR', + 'observed_make': 'VARCHAR', + 'observed_model': 'VARCHAR', + 'observed_model_year': 'VARCHAR' + } +) AS source +""" + + +class MartBuildError(RuntimeError): + """Raised when private input or a mart invariant fails closed.""" + + +@dataclass(frozen=True) +class InputBatch: + path: Path + manifest_path: Path + export_kind: str + start: datetime + end: datetime + rows: int + query_version: str + query_sha256: str + key_version: str + key_fingerprint: str + compressed_sha256: str + compressed_bytes: int + manifest: Mapping[str, Any] + + +@dataclass(frozen=True) +class BuildConfig: + input_path: Path = DEFAULT_INPUT + database_path: Path = DEFAULT_DATABASE + output_path: Path = DEFAULT_OUTPUT + episode_gap_days: int = 30 + memory_limit: str = "4GB" + threads: int = max(1, min(4, os.cpu_count() or 1)) + allow_gaps: bool = False + allow_external_output: bool = False + overwrite: bool = False + + +@dataclass(frozen=True) +class BuildSummary: + database_path: Path + output_path: Path + manifest_path: Path + input_rows: int + clean_events: int + episodes: int + mart_rows: int + eligible_rows: int + source_data_kind: str + population_estimate_allowed: bool + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def parse_manifest_timestamp(value: object, field: str) -> datetime: + if not isinstance(value, str): + raise MartBuildError(f"Manifest field {field!r} must be an ISO timestamp") + encoded = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = datetime.fromisoformat(encoded) + except ValueError as exc: + raise MartBuildError(f"Manifest field {field!r} is not ISO-8601") from exc + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) + return parsed + + +def _required_text(manifest: Mapping[str, Any], field: str) -> str: + value = manifest.get(field) + if not isinstance(value, str) or not value.strip(): + raise MartBuildError(f"Manifest field {field!r} must be non-empty text") + return value.strip() + + +def _required_nonnegative_int(manifest: Mapping[str, Any], field: str) -> int: + value = manifest.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise MartBuildError(f"Manifest field {field!r} must be a nonnegative integer") + return value + + +def _read_header(path: Path) -> Tuple[str, ...]: + try: + with gzip.open(path, "rt", encoding="utf-8", newline="") as handle: + row = next(csv.reader(handle), None) + except (OSError, UnicodeError, csv.Error) as exc: + raise MartBuildError(f"Cannot read gzip CSV header: {path}") from exc + if row is None: + raise MartBuildError(f"Input gzip CSV is empty: {path}") + return tuple(row) + + +def _load_batch(path: Path) -> InputBatch: + manifest_path = path.with_name(path.name + ".manifest.json") + if not manifest_path.is_file(): + raise MartBuildError(f"Missing paired manifest: {manifest_path}") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise MartBuildError(f"Cannot parse manifest: {manifest_path}") from exc + if not isinstance(manifest, dict): + raise MartBuildError(f"Manifest must contain a JSON object: {manifest_path}") + + columns = manifest.get("columns") + if not isinstance(columns, (list, tuple)) or tuple(columns) != EXPECTED_COLUMNS: + raise MartBuildError(f"Unexpected manifest columns: {manifest_path}") + if _read_header(path) != EXPECTED_COLUMNS: + raise MartBuildError(f"CSV header does not match the private schema: {path}") + + export_kind = _required_text(manifest, "export_kind") + if export_kind not in (BOUNDED_KIND, DEVELOPMENT_KIND): + raise MartBuildError(f"Unsupported export_kind {export_kind!r}: {manifest_path}") + start = parse_manifest_timestamp( + manifest.get("source_start_inclusive"), "source_start_inclusive" + ) + end = parse_manifest_timestamp( + manifest.get("source_end_exclusive"), "source_end_exclusive" + ) + if end <= start: + raise MartBuildError(f"Manifest source interval is empty or reversed: {path}") + + rows = _required_nonnegative_int(manifest, "rows") + compressed_bytes = _required_nonnegative_int(manifest, "compressed_file_bytes") + actual_bytes = path.stat().st_size + if compressed_bytes != actual_bytes: + raise MartBuildError( + f"Compressed byte count mismatch for {path}: " + f"manifest={compressed_bytes}, actual={actual_bytes}" + ) + compressed_sha256 = _required_text(manifest, "compressed_file_sha256").lower() + if not HEX_64.fullmatch(compressed_sha256): + raise MartBuildError(f"Invalid compressed SHA-256 in {manifest_path}") + actual_sha256 = file_sha256(path) + if compressed_sha256 != actual_sha256: + raise MartBuildError(f"Compressed SHA-256 mismatch for {path}") + + query_sha256 = _required_text(manifest, "query_sha256").lower() + if not HEX_64.fullmatch(query_sha256): + raise MartBuildError(f"Invalid query SHA-256 in {manifest_path}") + key_fingerprint = _required_text( + manifest, "vehicle_token_key_fingerprint" + ).lower() + if not FINGERPRINT.fullmatch(key_fingerprint): + raise MartBuildError(f"Invalid key fingerprint in {manifest_path}") + + if export_kind == BOUNDED_KIND: + if manifest.get("classification") != ( + "private_pseudonymized_analytical_staging" + ): + raise MartBuildError(f"Unexpected bounded-batch classification: {path}") + else: + if manifest.get("classification") != ( + "private_pseudonymized_development_only" + ): + raise MartBuildError(f"Unexpected development classification: {path}") + if manifest.get("population_estimate_allowed") is not False: + raise MartBuildError( + "Development history manifest must prohibit population estimates" + ) + + return InputBatch( + path=path, + manifest_path=manifest_path, + export_kind=export_kind, + start=start, + end=end, + rows=rows, + query_version=_required_text(manifest, "query_version"), + query_sha256=query_sha256, + key_version=_required_text(manifest, "vehicle_token_key_version"), + key_fingerprint=key_fingerprint, + compressed_sha256=compressed_sha256, + compressed_bytes=compressed_bytes, + manifest=manifest, + ) + + +def _discover_files(input_path: Path) -> List[Path]: + if input_path.is_file(): + if not input_path.name.endswith(".csv.gz"): + raise MartBuildError("A feature-mart input file must end in .csv.gz") + return [input_path] + if not input_path.is_dir(): + raise MartBuildError(f"Input path does not exist: {input_path}") + paths = sorted(path for path in input_path.glob("*.csv.gz") if path.is_file()) + if not paths: + raise MartBuildError(f"No .csv.gz inputs found in {input_path}") + return paths + + +def discover_and_validate_batches(config: BuildConfig) -> List[InputBatch]: + input_path = config.input_path.resolve() + if not config.allow_external_output and not input_path.is_relative_to( + PRIVATE_DATA_ROOT + ): + raise MartBuildError(f"Private inputs must stay under {PRIVATE_DATA_ROOT}") + + batches = [_load_batch(path.resolve()) for path in _discover_files(input_path)] + kinds = {batch.export_kind for batch in batches} + if len(kinds) != 1: + raise MartBuildError("Do not mix bounded batches and development samples") + kind = next(iter(kinds)) + + key_versions = {batch.key_version for batch in batches} + fingerprints = {batch.key_fingerprint for batch in batches} + if len(key_versions) != 1 or len(fingerprints) != 1: + raise MartBuildError("All inputs must use one VIN-token key and version") + + if kind == DEVELOPMENT_KIND: + if len(batches) != 1: + raise MartBuildError("A development mart accepts exactly one history sample") + return batches + + query_versions = {batch.query_version for batch in batches} + query_hashes = {batch.query_sha256 for batch in batches} + if len(query_versions) != 1 or len(query_hashes) != 1: + raise MartBuildError("All bounded batches must use one extraction query version") + + batches.sort(key=lambda batch: (batch.start, batch.end, str(batch.path))) + for previous, current in zip(batches, batches[1:]): + if current.start < previous.end: + raise MartBuildError( + f"Source intervals overlap: {previous.path.name} and " + f"{current.path.name}" + ) + if current.start > previous.end and not config.allow_gaps: + raise MartBuildError( + f"Source interval gap between {previous.end.isoformat()} and " + f"{current.start.isoformat()}; use --allow-gaps only for an " + "explicitly incomplete analytical build" + ) + return batches + + +def _validate_config(config: BuildConfig) -> BuildConfig: + if not 1 <= config.episode_gap_days <= 365: + raise MartBuildError("episode_gap_days must be between 1 and 365") + if not 1 <= config.threads <= 64: + raise MartBuildError("threads must be between 1 and 64") + if not MEMORY_LIMIT.fullmatch(config.memory_limit): + raise MartBuildError("memory_limit must look like 4096MB or 4GB") + + database_path = config.database_path.resolve() + output_path = config.output_path.resolve() + if output_path.suffix.lower() != ".parquet": + raise MartBuildError("Feature-mart output must use a .parquet suffix") + if not config.allow_external_output: + for path in (database_path, output_path): + if not path.is_relative_to(PRIVATE_DATA_ROOT): + raise MartBuildError(f"Private outputs must stay under {PRIVATE_DATA_ROOT}") + if database_path == output_path: + raise MartBuildError("Database and Parquet output paths must differ") + + return BuildConfig( + input_path=config.input_path.resolve(), + database_path=database_path, + output_path=output_path, + episode_gap_days=config.episode_gap_days, + memory_limit=config.memory_limit.upper(), + threads=config.threads, + allow_gaps=config.allow_gaps, + allow_external_output=config.allow_external_output, + overwrite=config.overwrite, + ) + + +def _load_sql(name: str) -> str: + path = SQL_ROOT / name + try: + return path.read_text(encoding="utf-8") + except OSError as exc: + raise MartBuildError(f"Cannot read local transform SQL: {path}") from exc + + +def _remove_partial(paths: Iterable[Path]) -> None: + for path in paths: + path.unlink(missing_ok=True) + + +def _output_columns(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, str]]: + rows = connection.execute("DESCRIBE SELECT * FROM feature_mart").fetchall() + return [ + {"name": str(row[0]), "duckdb_type": str(row[1])} + for row in rows + ] + + +def build_feature_mart(config: BuildConfig) -> BuildSummary: + config = _validate_config(config) + batches = discover_and_validate_batches(config) + source_data_kind = batches[0].export_kind + population_estimate_allowed = ( + source_data_kind == BOUNDED_KIND and not config.allow_gaps + ) + + database_path = config.database_path + output_path = config.output_path + manifest_path = output_path.with_name(output_path.name + ".manifest.json") + partial_database = database_path.with_name(database_path.name + ".partial") + partial_output = output_path.with_name(output_path.name + ".partial") + partial_manifest = manifest_path.with_name(manifest_path.name + ".partial") + partial_wal = partial_database.with_name(partial_database.name + ".wal") + partial_paths = (partial_database, partial_output, partial_manifest, partial_wal) + + finals = (database_path, output_path, manifest_path) + existing = [path for path in finals if path.exists()] + if existing and not config.overwrite: + raise MartBuildError( + "Refusing to overwrite existing outputs: " + + ", ".join(str(path) for path in existing) + ) + + os.umask(0o077) + database_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + output_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temp_directory = database_path.parent / ".duckdb_tmp" + temp_directory.mkdir(parents=True, exist_ok=True, mode=0o700) + _remove_partial(partial_paths) + + connection: Optional[duckdb.DuckDBPyConnection] = None + try: + connection = duckdb.connect(str(partial_database)) + connection.execute("SET threads = ?", [config.threads]) + connection.execute("SET memory_limit = ?", [config.memory_limit]) + connection.execute("SET temp_directory = ?", [str(temp_directory)]) + connection.execute("SET preserve_insertion_order = false") + connection.execute( + """ + CREATE TABLE build_config AS + SELECT + ?::INTEGER AS episode_gap_days, + ?::VARCHAR AS source_data_kind, + ?::BOOLEAN AS population_estimate_allowed + """, + [ + config.episode_gap_days, + source_data_kind, + population_estimate_allowed, + ], + ) + connection.execute(CREATE_STAGING_SQL) + + for batch in batches: + connection.execute( + INGEST_SQL, + [ + str(batch.path), + batch.export_kind, + batch.start.isoformat(sep=" "), + batch.end.isoformat(sep=" "), + str(batch.path), + ], + ) + + actual_by_file = dict( + connection.execute( + "SELECT batch_file, count(*) FROM stg_events GROUP BY batch_file" + ).fetchall() + ) + for batch in batches: + actual = int(actual_by_file.get(str(batch.path), 0)) + if actual != batch.rows: + raise MartBuildError( + f"CSV row count mismatch for {batch.path}: " + f"manifest={batch.rows}, actual={actual}" + ) + + duplicate_ids = connection.execute( + """ + SELECT count(*) + FROM ( + SELECT internal_event_id + FROM stg_events + GROUP BY internal_event_id + HAVING count(*) > 1 + ) + """ + ).fetchone()[0] + if duplicate_ids: + raise MartBuildError( + f"Input contains {duplicate_ids} duplicated internal event IDs" + ) + + inconsistent_buckets = connection.execute( + """ + SELECT count(*) + FROM ( + SELECT lower(vehicle_token) AS vehicle_token + FROM stg_events + WHERE regexp_full_match(lower(vehicle_token), '^[0-9a-f]{64}$') + AND try_cast(vehicle_bucket AS INTEGER) BETWEEN 0 AND 99 + GROUP BY lower(vehicle_token) + HAVING count(DISTINCT try_cast(vehicle_bucket AS INTEGER)) > 1 + ) + """ + ).fetchone()[0] + if inconsistent_buckets: + raise MartBuildError( + f"Input contains {inconsistent_buckets} vehicle tokens with " + "inconsistent audit buckets" + ) + + sql_hashes: Dict[str, str] = {} + for name in SQL_FILES: + sql = _load_sql(name) + sql_hashes[name] = hashlib.sha256(sql.encode("utf-8")).hexdigest() + connection.execute(sql) + + violations = connection.execute( + """ + SELECT check_name, violation_count + FROM mart_validation + WHERE violation_count <> 0 + ORDER BY check_name + """ + ).fetchall() + if violations: + details = ", ".join(f"{name}={count}" for name, count in violations) + raise MartBuildError(f"Mart invariant failure: {details}") + + counts = connection.execute( + """ + SELECT + (SELECT count(*) FROM stg_events), + (SELECT count(*) FROM clean_events), + (SELECT count(*) FROM inspection_episodes), + (SELECT count(*) FROM feature_mart), + (SELECT count(*) FROM feature_mart + WHERE eligible_returning_target) + """ + ).fetchone() + input_rows, clean_events, episodes, mart_rows, eligible_rows = map( + int, counts + ) + + columns = _output_columns(connection) + connection.execute("ANALYZE") + connection.execute( + """ + COPY ( + SELECT * + FROM feature_mart + ORDER BY vehicle_token, episode_number + ) TO ? (FORMAT PARQUET, COMPRESSION ZSTD) + """, + [str(partial_output)], + ) + + build_id = hashlib.sha256( + json.dumps( + { + "inputs": [batch.compressed_sha256 for batch in batches], + "sql": sql_hashes, + "gap": config.episode_gap_days, + }, + sort_keys=True, + ).encode("utf-8") + ).hexdigest()[:16] + output_sha256 = file_sha256(partial_output) + build_manifest: Dict[str, Any] = { + "build_kind": "private_leakage_safe_inspection_feature_mart", + "build_id": build_id, + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "classification": "private_pseudonymized_analytical_mart", + "source_data_kind": source_data_kind, + "population_estimate_allowed": population_estimate_allowed, + "episode_gap_days": config.episode_gap_days, + "vehicle_quality_contract": ( + "eligibility uses only prior events: at most 50 prior events " + "and at most 4 prior events on any day" + ), + "outcome_label_source_contract": { + "allowed_values": ["overall_result", "utah_obd_proxy"], + "null_allowed_only_for_unlabeled_outcomes": True, + "utah_obd_proxy_source_era": "utah", + "feature_role": "audit_only_not_a_predictor", + }, + "vin_audit_contract": "vehicle_bucket < 10", + "inputs": [ + { + "file": str(batch.path), + "manifest": str(batch.manifest_path), + "export_kind": batch.export_kind, + "query_version": batch.query_version, + "query_sha256": batch.query_sha256, + "source_start_inclusive": batch.start.isoformat(), + "source_end_exclusive": batch.end.isoformat(), + "rows": batch.rows, + "compressed_file_sha256": batch.compressed_sha256, + } + for batch in batches + ], + "vehicle_token_key_version": batches[0].key_version, + "vehicle_token_key_fingerprint": batches[0].key_fingerprint, + "transform_sql_sha256": sql_hashes, + "row_counts": { + "input": input_rows, + "clean_events": clean_events, + "episodes": episodes, + "feature_mart": mart_rows, + "eligible_returning_targets": eligible_rows, + }, + "columns": columns, + "parquet_sha256": output_sha256, + "parquet_bytes": partial_output.stat().st_size, + } + partial_manifest.write_text( + json.dumps(build_manifest, indent=2) + "\n", encoding="utf-8" + ) + + connection.execute("CHECKPOINT") + connection.close() + connection = None + + os.replace(partial_database, database_path) + os.replace(partial_output, output_path) + os.replace(partial_manifest, manifest_path) + + return BuildSummary( + database_path=database_path, + output_path=output_path, + manifest_path=manifest_path, + input_rows=input_rows, + clean_events=clean_events, + episodes=episodes, + mart_rows=mart_rows, + eligible_rows=eligible_rows, + source_data_kind=source_data_kind, + population_estimate_allowed=population_estimate_allowed, + ) + except Exception: + if connection is not None: + connection.close() + _remove_partial(partial_paths) + raise + + +def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--input", + "--batch-dir", + dest="input_path", + type=Path, + default=DEFAULT_INPUT, + help="A bounded-batch directory or one development history CSV.gz.", + ) + parser.add_argument("--database", type=Path, default=DEFAULT_DATABASE) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--episode-gap-days", type=int, default=30) + parser.add_argument("--memory-limit", default="4GB") + parser.add_argument( + "--threads", type=int, default=max(1, min(4, os.cpu_count() or 1)) + ) + parser.add_argument("--allow-gaps", action="store_true") + parser.add_argument("--allow-external-output", action="store_true") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args(argv) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parse_args(argv) + config = BuildConfig( + input_path=args.input_path, + database_path=args.database, + output_path=args.output, + episode_gap_days=args.episode_gap_days, + memory_limit=args.memory_limit, + threads=args.threads, + allow_gaps=args.allow_gaps, + allow_external_output=args.allow_external_output, + overwrite=args.overwrite, + ) + try: + summary = build_feature_mart(config) + except (MartBuildError, duckdb.Error, OSError) as exc: + print(f"Feature-mart build failed: {exc}", file=sys.stderr) + return 1 + + print( + f"Built {summary.mart_rows:,} episode rows " + f"({summary.eligible_rows:,} eligible returning targets)" + ) + print(f"Private DuckDB warehouse: {summary.database_path}") + print(f"Private Parquet mart: {summary.output_path}") + print(f"Build manifest: {summary.manifest_path}") + if not summary.population_estimate_allowed: + print("Development sample: population estimates are prohibited.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_connection.py b/scripts/check_connection.py new file mode 100644 index 0000000..3ded5a5 --- /dev/null +++ b/scripts/check_connection.py @@ -0,0 +1,96 @@ +"""Verify the countydata connection without exposing credentials or row data.""" + +from __future__ import annotations + +import os +import sys + +import psycopg2 + + +REQUIRED_ENV_VARS = ( + "PGHOST", + "PGPORT", + "PGDATABASE", + "PGUSER", + "PGPASSWORD", + "PGSSLMODE", +) + + +def main() -> int: + missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name)] + if missing: + print( + "Missing database environment variables: " + ", ".join(missing), + file=sys.stderr, + ) + print("Run this through the VS Code task so .env is loaded safely.", file=sys.stderr) + return 1 + + if os.environ["PGSSLMODE"].lower() not in {"require", "verify-ca", "verify-full"}: + print("Refusing to connect because PGSSLMODE does not require TLS.", file=sys.stderr) + return 1 + + try: + connection = psycopg2.connect( + application_name="summer_project_vscode_check", + connect_timeout=10, + options=( + "-c default_transaction_read_only=on " + "-c statement_timeout=30000" + ), + ) + connection.set_session(readonly=True) + except psycopg2.Error as exc: + print(f"Connection failed: {exc.diag.message_primary or type(exc).__name__}", file=sys.stderr) + return 1 + + try: + with connection, connection.cursor() as cursor: + cursor.execute( + """ + SELECT + current_database(), + current_user, + current_setting('transaction_read_only'), + version() + """ + ) + database, user, read_only, version = cursor.fetchone() + + cursor.execute( + """ + SELECT ssl, version, cipher, bits + FROM pg_stat_ssl + WHERE pid = pg_backend_pid() + """ + ) + ssl, tls_version, cipher, bits = cursor.fetchone() + + cursor.execute( + """ + SELECT count(*) + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + """ + ) + relation_count = cursor.fetchone()[0] + + print("Countydata connection succeeded") + print(f" database: {database}") + print(f" user: {user}") + print(f" transaction read-only: {read_only}") + print(f" TLS: {ssl} ({tls_version}, {cipher}, {bits}-bit)") + print(f" visible non-system relations: {relation_count}") + print(f" server: {version}") + return 0 + finally: + connection.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/create_vin_hash_key.py b/scripts/create_vin_hash_key.py new file mode 100644 index 0000000..b4d2da1 --- /dev/null +++ b/scripts/create_vin_hash_key.py @@ -0,0 +1,56 @@ +"""Create the local project-specific VIN HMAC key without printing it.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import secrets +import stat +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_PATH = PROJECT_ROOT / ".secrets/vin_hmac.key" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--path", type=Path, default=DEFAULT_PATH) + return parser.parse_args() + + +def key_fingerprint(key: bytes) -> str: + return hashlib.sha256( + b"utah-vehicle-health/key-fingerprint/v1\0" + key + ).hexdigest()[:16] + + +def main() -> int: + path = parse_args().path.resolve() + os.umask(0o077) + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + + if path.exists(): + mode = stat.S_IMODE(path.stat().st_mode) + if mode & 0o077: + print(f"Existing key has unsafe permissions: {path}", file=sys.stderr) + return 1 + print(f"Key already exists; leaving it unchanged: {path}") + return 0 + + key = secrets.token_bytes(32) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="ascii") as handle: + handle.write(key.hex() + "\n") + handle.flush() + os.fsync(handle.fileno()) + + print(f"Created private VIN HMAC key: {path}") + print(f"Non-secret key fingerprint: {key_fingerprint(key)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export_dashboard_data.py b/scripts/export_dashboard_data.py new file mode 100644 index 0000000..f9c9277 --- /dev/null +++ b/scripts/export_dashboard_data.py @@ -0,0 +1,1205 @@ +"""Publish privacy-reviewed aggregate JSON assets for the dashboard. + +The exporter never loads a model or emits row-level predictions. It reads only +the train, tune, and calibration portions of the private feature mart. The 2025 +locked-test partition is outside every aggregation query. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +import duckdb + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +PUBLIC_DATA_ROOT = (PROJECT_ROOT / "dashboard/public/data").resolve() +SCHEMA_VERSION = "dashboard_data_v1" +ALLOWED_PARTITIONS = ("train", "tune", "calibrate") +PARTITION_ORDER = {name: index for index, name in enumerate(ALLOWED_PARTITIONS)} +LOCKED_PARTITION_ALIASES = ("test", "locked_test", "locked-test") +MIN_SUPPORT = 100 +MIN_CLASS_COUNT = 10 +SUPPORT_ROUNDING = 100 +MAX_SCORECARDS = 200 + +ROW_ASSET_SCHEMAS = { + "overview_period_county.json": ( + "year", + "quarter", + "public_county", + "support_rounded", + "nonpass_rate", + ), + "age_risk_curve.json": ( + "age_band", + "support_rounded", + "nonpass_rate", + ), + "cohort_scorecard.json": ( + "prior_make", + "prior_model", + "support_rounded", + "nonpass_rate", + ), + "model_diagnostics.json": ( + "model", + "partition", + "average_precision", + "brier", + "log_loss", + "roc_auc", + "top_10_capture", + ), + "coverage_quality.json": ( + "year", + "source_era", + "support_rounded", + "labeled_rate", + "overall_result_share", + "utah_obd_proxy_share", + ), +} +PUBLIC_ASSET_NAMES = tuple( + sorted( + list(ROW_ASSET_SCHEMAS) + + ["filter_catalog.json", "data_manifest.json"] + ) +) +SHA256_MANIFEST_NAME = "sha256_manifest.json" +PUBLIC_OUTPUT_NAMES = frozenset(PUBLIC_ASSET_NAMES + (SHA256_MANIFEST_NAME,)) + +REQUIRED_MART_COLUMNS = { + "vehicle_token", + "is_vin_audit", + "episode_number", + "episode_start", + "first_outcome", + "target_outcome_label_source", + "target_nonpass", + "eligible_returning_target", + "temporal_partition", + "public_county", + "source_era", + "vehicle_age", + "last_observed_make", + "last_observed_model", +} +DENIED_EXACT_KEYS = { + "vehicle_token", + "vehicle_bucket", + "is_vin_audit", + "episode_number", + "episode_start", + "first_outcome", + "target_nonpass", + "target_outcome_label_source", + "outcome_label_source", + "internal_event_id", + "vin", + "plate", + "zip", + "zipcode", + "station", + "station_id", + "raw_json", + "row_prediction", + "prediction", + "probability", + "pass_count", + "nonpass_count", + "row_count", +} +DENIED_KEY_FRAGMENTS = ( + "vehicle_token", + "internal_event", + "license_plate", + "exact_timestamp", + "row_prediction", + "raw_", + "label_source", +) +EXACT_TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}") +VIN_LIKE_PATTERN = re.compile(r"^[A-HJ-NPR-Z0-9]{17}$") +SAFE_CATEGORY_PATTERN = re.compile(r"^[^\x00-\x1f\x7f]{1,80}$") +SAFE_MODEL_NAME_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,64}$") + + +class DashboardExportError(Exception): + """Base class for expected publication failures.""" + + +class ManifestError(DashboardExportError): + """Raised when private input provenance cannot be verified.""" + + +class PublicSchemaError(DashboardExportError): + """Raised when an asset violates the public-data contract.""" + + +@dataclass(frozen=True) +class ModelBundle: + manifest: Path + metrics: Path + + +def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mart", required=True, type=Path) + parser.add_argument("--mart-manifest", type=Path) + parser.add_argument( + "--model-manifest", + required=True, + action="append", + type=Path, + help="Private model manifest; repeat once per model bundle.", + ) + parser.add_argument( + "--model-metrics", + required=True, + action="append", + type=Path, + help="Metrics JSON paired by position with --model-manifest.", + ) + parser.add_argument("--output-dir", type=Path, default=PUBLIC_DATA_ROOT) + parser.add_argument("--development-preview", action="store_true") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args(argv) + + +def _reject_json_constant(value: str) -> object: + raise ManifestError("JSON contains a non-finite numeric constant: " + value) + + +def _unique_object(pairs: Iterable[Tuple[str, object]]) -> Dict[str, object]: + result: Dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ManifestError("JSON contains duplicate key: " + key) + result[key] = value + return result + + +def load_json(path: Path) -> object: + try: + if path.stat().st_size > 20 * 1024 * 1024: + raise ManifestError("Refusing an unexpectedly large JSON input") + return json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_json_constant, + ) + except OSError as exc: + raise ManifestError("Cannot read required JSON input") from exc + except UnicodeError as exc: + raise ManifestError("JSON input is not valid UTF-8") from exc + except json.JSONDecodeError as exc: + raise ManifestError("JSON input is malformed") from exc + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as exc: + raise ManifestError("Cannot read a required private input") from exc + return digest.hexdigest() + + +def _require_mapping(value: object, label: str) -> Mapping[str, object]: + if not isinstance(value, dict): + raise ManifestError(label + " must be a JSON object") + return value + + +def _mart_manifest_path(mart: Path, explicit: Optional[Path]) -> Path: + if explicit is not None: + return explicit + return mart.with_name(mart.name + ".manifest.json") + + +def validate_mart_manifest( + mart: Path, + manifest_path: Path, + development_preview: bool, +) -> Tuple[Mapping[str, object], bool, str]: + manifest = _require_mapping(load_json(manifest_path), "Mart manifest") + if manifest.get("build_kind") != "private_leakage_safe_inspection_feature_mart": + raise ManifestError("Input is not the approved private feature mart") + classification = manifest.get("classification") + if not isinstance(classification, str) or not classification.startswith("private_"): + raise ManifestError("Feature mart classification is not private") + + expected_digest = manifest.get("parquet_sha256") + if not isinstance(expected_digest, str) or not re.fullmatch( + r"[0-9a-f]{64}", expected_digest + ): + raise ManifestError("Mart manifest has no valid Parquet digest") + actual_digest = file_sha256(mart) + if actual_digest != expected_digest: + raise ManifestError("Feature mart digest does not match its manifest") + + population_allowed = manifest.get("population_estimate_allowed") + if not isinstance(population_allowed, bool): + raise ManifestError("Mart population-estimate flag must be boolean") + source_kind = manifest.get("source_data_kind") + if not isinstance(source_kind, str): + raise ManifestError("Mart source-data kind is missing") + source_kind_lower = source_kind.lower() + development_source = ( + not population_allowed + or "development" in source_kind_lower + or "sample" in source_kind_lower + ) + if development_source and not development_preview: + raise ManifestError( + "Development-sample marts require --development-preview" + ) + if not development_preview and not population_allowed: + raise ManifestError("Population publication requires an approved mart") + + columns = manifest.get("columns") + if not isinstance(columns, list): + raise ManifestError("Mart manifest has no column contract") + declared_names = { + item.get("name") + for item in columns + if isinstance(item, dict) and isinstance(item.get("name"), str) + } + missing = sorted(REQUIRED_MART_COLUMNS - declared_names) + if missing: + raise ManifestError("Mart manifest is missing required columns") + return manifest, development_source, actual_digest + + +def load_model_diagnostics( + bundles: Sequence[ModelBundle], + mart_digest: str, + partition_vehicle_support: Mapping[str, Tuple[int, int]], +) -> Tuple[List[Dict[str, object]], List[Tuple[str, str]]]: + diagnostics: List[Dict[str, object]] = [] + model_provenance: List[Tuple[str, str]] = [] + seen = set() + for bundle in bundles: + manifest = _require_mapping(load_json(bundle.manifest), "Model manifest") + if manifest.get("locked_test_evaluated") is not False: + raise ManifestError( + "Every model manifest must state locked_test_evaluated=false" + ) + classification = manifest.get("classification") + if not isinstance(classification, str) or "no_row_predictions" not in classification: + raise ManifestError("Model artifact classification is not publishable") + if manifest.get("input_sha256") != mart_digest: + raise ManifestError("Model metrics were not built from this mart") + expected_metrics_digest = manifest.get("metrics_sha256") + if not isinstance(expected_metrics_digest, str) or not re.fullmatch( + r"[0-9a-f]{64}", expected_metrics_digest + ): + raise ManifestError("Model manifest has no valid metrics digest") + metrics_digest = file_sha256(bundle.metrics) + if metrics_digest != expected_metrics_digest: + raise ManifestError("Model metrics digest does not match its manifest") + model_version = manifest.get("model_version") + if not isinstance(model_version, str) or not SAFE_MODEL_NAME_PATTERN.fullmatch( + model_version + ): + raise ManifestError("Model manifest has an unsafe model version") + model_provenance.append((model_version, metrics_digest)) + + payload = load_json(bundle.metrics) + if isinstance(payload, dict): + payload = payload.get("metrics") + if not isinstance(payload, list): + raise ManifestError("Model metrics must be a JSON array") + + for raw in payload: + if not isinstance(raw, dict): + raise ManifestError("Each model metric must be an object") + partition = raw.get("partition") + if partition not in ALLOWED_PARTITIONS: + raise ManifestError( + "Metrics contain a locked, shadow, or unknown partition" + ) + if raw.get("cohort") != "non_audit": + continue + episodes = raw.get("episodes") + nonpass = raw.get("nonpass") + vehicles = raw.get("vehicles") + if ( + isinstance(episodes, bool) + or not isinstance(episodes, int) + or isinstance(nonpass, bool) + or not isinstance(nonpass, int) + or isinstance(vehicles, bool) + or not isinstance(vehicles, int) + or vehicles < 0 + or vehicles > episodes + or nonpass < 0 + or nonpass > episodes + ): + raise ManifestError( + "Model metrics lack valid private suppression counts" + ) + class_vehicle_support = partition_vehicle_support.get(partition) + if class_vehicle_support is None or not _publishable( + episodes, + nonpass, + vehicles, + class_vehicle_support[0], + class_vehicle_support[1], + ): + continue + model = raw.get("model") + if not isinstance(model, str) or not SAFE_MODEL_NAME_PATTERN.fullmatch(model): + raise ManifestError("Model metric has an unsafe model name") + + row: Dict[str, object] = {"model": model, "partition": partition} + for key in ( + "average_precision", + "brier", + "log_loss", + "roc_auc", + "top_10_capture", + ): + value = raw.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ManifestError("Model metric is missing a numeric value") + number = float(value) + if not math.isfinite(number) or number < 0.0: + raise ManifestError("Model metric is non-finite or negative") + if key != "log_loss" and number > 1.0: + raise ManifestError("Bounded model metric exceeds one") + row[key] = round(number, 4) + + identity = (model, partition) + if identity in seen: + raise ManifestError("Duplicate model/partition diagnostic") + seen.add(identity) + diagnostics.append(row) + + if not diagnostics: + raise ManifestError("No approved train/tune/calibrate diagnostics found") + diagnostics.sort( + key=lambda row: (str(row["model"]), PARTITION_ORDER[str(row["partition"])]) + ) + model_provenance.sort() + return diagnostics, model_provenance + + +def _round_support(value: int) -> int: + if value < MIN_SUPPORT: + raise PublicSchemaError("Unsuppressed support is below the threshold") + return int(math.floor((value + SUPPORT_ROUNDING / 2) / SUPPORT_ROUNDING)) * SUPPORT_ROUNDING + + +def _rate(numerator: int, denominator: int) -> float: + if denominator <= 0 or numerator < 0 or numerator > denominator: + raise PublicSchemaError("Invalid aggregate rate inputs") + return round(numerator / denominator, 3) + + +def _publishable( + n: int, + nonpass: int, + vehicles: int, + pass_vehicles: int, + nonpass_vehicles: int, +) -> bool: + return ( + n >= MIN_SUPPORT + and nonpass >= MIN_CLASS_COUNT + and n - nonpass >= MIN_CLASS_COUNT + and vehicles >= MIN_SUPPORT + and pass_vehicles >= MIN_CLASS_COUNT + and nonpass_vehicles >= MIN_CLASS_COUNT + ) + + +def _safe_category(value: object, label: str) -> str: + if not isinstance(value, str): + raise PublicSchemaError(label + " category is not text") + normalized = value.strip() + if not SAFE_CATEGORY_PATTERN.fullmatch(normalized): + raise PublicSchemaError(label + " category contains unsafe characters") + compact = re.sub(r"[^A-Za-z0-9]", "", normalized).upper() + if VIN_LIKE_PATTERN.fullmatch(compact): + raise PublicSchemaError(label + " category resembles a VIN") + return normalized + + +def _create_safe_mart(connection: duckdb.DuckDBPyConnection, mart: Path) -> None: + description = connection.execute( + "DESCRIBE SELECT * FROM read_parquet(?)", [str(mart)] + ).fetchall() + actual_columns = {str(row[0]) for row in description} + if not REQUIRED_MART_COLUMNS.issubset(actual_columns): + raise ManifestError("Parquet schema does not match the mart contract") + + boundary_errors = connection.execute( + """ + SELECT count(*) + FROM read_parquet(?) + WHERE (temporal_partition IN ('train', 'tune', 'calibrate') + AND episode_start >= TIMESTAMP '2025-01-01') + OR (temporal_partition IN ('test', 'locked_test', 'locked-test') + AND episode_start < TIMESTAMP '2025-01-01') + """, + [str(mart)], + ).fetchone()[0] + if boundary_errors: + raise ManifestError("Mart partition timestamps violate the locked boundary") + + # The filter is applied before target fields enter this temporary table. + # No subsequent query can read or evaluate a 2025 target. + connection.execute( + """ + CREATE TEMP TABLE safe_mart AS + SELECT + vehicle_token, + is_vin_audit, + episode_number, + episode_start, + first_outcome, + target_outcome_label_source, + target_nonpass, + eligible_returning_target, + temporal_partition, + public_county, + source_era, + vehicle_age, + last_observed_make, + last_observed_model + FROM read_parquet(?) + WHERE temporal_partition IN ('train', 'tune', 'calibrate') + AND episode_start < TIMESTAMP '2025-01-01' + """, + [str(mart)], + ) + + +def _partition_vehicle_support( + connection: duckdb.DuckDBPyConnection, +) -> Dict[str, Tuple[int, int]]: + """Return private class-level entity supports for diagnostic suppression.""" + raw = connection.execute( + """ + SELECT temporal_partition, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 0 + )::BIGINT AS pass_vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 1 + )::BIGINT AS nonpass_vehicles + FROM safe_mart + WHERE eligible_returning_target + AND target_nonpass IN (0, 1) + AND NOT is_vin_audit + GROUP BY 1 + """ + ).fetchall() + return { + str(partition): (int(pass_vehicles), int(nonpass_vehicles)) + for partition, pass_vehicles, nonpass_vehicles in raw + } + + +def _overview_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, object]]: + raw = connection.execute( + """ + SELECT year(episode_start)::INTEGER AS year, + quarter(episode_start)::INTEGER AS quarter, + public_county, + count(*)::BIGINT AS n, + sum(target_nonpass)::BIGINT AS nonpass, + count(DISTINCT vehicle_token)::BIGINT AS vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 0 + )::BIGINT AS pass_vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 1 + )::BIGINT AS nonpass_vehicles + FROM safe_mart + WHERE eligible_returning_target + AND target_nonpass IN (0, 1) + AND public_county IS NOT NULL + GROUP BY 1, 2, 3 + ORDER BY 1, 2, 3 + """ + ).fetchall() + rows = [] + for ( + year, + quarter, + county, + n, + nonpass, + vehicles, + pass_vehicles, + nonpass_vehicles, + ) in raw: + if not _publishable( + n, nonpass, vehicles, pass_vehicles, nonpass_vehicles + ): + continue + rows.append( + { + "year": int(year), + "quarter": int(quarter), + "public_county": _safe_category(county, "County"), + "support_rounded": _round_support(n), + "nonpass_rate": _rate(nonpass, n), + } + ) + return rows + + +AGE_BAND_SQL = """CASE + WHEN vehicle_age BETWEEN 0 AND 3 THEN '0-3' + WHEN vehicle_age BETWEEN 4 AND 7 THEN '4-7' + WHEN vehicle_age BETWEEN 8 AND 11 THEN '8-11' + WHEN vehicle_age BETWEEN 12 AND 15 THEN '12-15' + WHEN vehicle_age BETWEEN 16 AND 20 THEN '16-20' + WHEN vehicle_age >= 21 THEN '21+' + ELSE NULL +END""" +AGE_BAND_ORDER = {name: index for index, name in enumerate( + ("0-3", "4-7", "8-11", "12-15", "16-20", "21+") +)} + + +def _age_risk_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, object]]: + raw = connection.execute( + f""" + SELECT {AGE_BAND_SQL} AS age_band, + count(*)::BIGINT AS n, + sum(target_nonpass)::BIGINT AS nonpass, + count(DISTINCT vehicle_token)::BIGINT AS vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 0 + )::BIGINT AS pass_vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 1 + )::BIGINT AS nonpass_vehicles + FROM safe_mart + WHERE eligible_returning_target AND target_nonpass IN (0, 1) + GROUP BY 1 + ORDER BY 1 + """ + ).fetchall() + rows = [] + for ( + age_band, + n, + nonpass, + vehicles, + pass_vehicles, + nonpass_vehicles, + ) in raw: + if age_band is None or not _publishable( + n, nonpass, vehicles, pass_vehicles, nonpass_vehicles + ): + continue + if age_band not in AGE_BAND_ORDER: + raise PublicSchemaError("Unexpected age band") + rows.append( + { + "age_band": age_band, + "support_rounded": _round_support(n), + "nonpass_rate": _rate(nonpass, n), + } + ) + rows.sort(key=lambda row: AGE_BAND_ORDER[str(row["age_band"])]) + return rows + + +def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, object]]: + raw = connection.execute( + """ + SELECT last_observed_make, last_observed_model, + count(*)::BIGINT AS n, + sum(target_nonpass)::BIGINT AS nonpass, + count(DISTINCT vehicle_token)::BIGINT AS vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 0 + )::BIGINT AS pass_vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 1 + )::BIGINT AS nonpass_vehicles + FROM safe_mart + WHERE eligible_returning_target + AND target_nonpass IN (0, 1) + AND last_observed_make IS NOT NULL + AND last_observed_model IS NOT NULL + GROUP BY 1, 2 + HAVING count(*) >= 100 + AND sum(target_nonpass) >= 10 + AND count(*) - sum(target_nonpass) >= 10 + AND count(DISTINCT vehicle_token) >= 100 + AND count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 0 + ) >= 10 + AND count(DISTINCT vehicle_token) FILTER ( + WHERE target_nonpass = 1 + ) >= 10 + ORDER BY n DESC, last_observed_make, last_observed_model + LIMIT 200 + """ + ).fetchall() + rows = [] + for ( + make, + model, + n, + nonpass, + vehicles, + pass_vehicles, + nonpass_vehicles, + ) in raw: + if not _publishable( + n, nonpass, vehicles, pass_vehicles, nonpass_vehicles + ): + continue + try: + safe_make = _safe_category(make, "Make") + safe_model = _safe_category(model, "Model") + except PublicSchemaError: + continue + rows.append( + { + "prior_make": safe_make, + "prior_model": safe_model, + "support_rounded": _round_support(n), + "nonpass_rate": _rate(nonpass, n), + } + ) + rows.sort(key=lambda row: (str(row["prior_make"]), str(row["prior_model"]))) + return rows + + +def _coverage_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, object]]: + raw = connection.execute( + """ + SELECT year(episode_start)::INTEGER AS year, + source_era, + count(*)::BIGINT AS n, + count(*) FILTER ( + WHERE first_outcome IN ('pass','fail','reject','abort') + )::BIGINT AS labeled, + count(*) FILTER (WHERE first_outcome = 'pass')::BIGINT AS pass, + count(*) FILTER ( + WHERE first_outcome IN ('fail','reject','abort') + )::BIGINT AS nonpass, + count(*) FILTER ( + WHERE target_outcome_label_source = 'overall_result' + )::BIGINT AS overall_labels, + count(*) FILTER ( + WHERE target_outcome_label_source = 'utah_obd_proxy' + )::BIGINT AS proxy_labels, + count(DISTINCT vehicle_token)::BIGINT AS vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE first_outcome = 'pass' + )::BIGINT AS pass_vehicles, + count(DISTINCT vehicle_token) FILTER ( + WHERE first_outcome IN ('fail','reject','abort') + )::BIGINT AS nonpass_vehicles + FROM safe_mart + WHERE episode_number > 1 AND source_era IS NOT NULL + GROUP BY 1, 2 + ORDER BY 1, 2 + """ + ).fetchall() + rows = [] + for ( + year, + source, + n, + labeled, + passes, + nonpass, + overall, + proxy, + vehicles, + pass_vehicles, + nonpass_vehicles, + ) in raw: + if not _publishable( + n, nonpass, vehicles, pass_vehicles, nonpass_vehicles + ) or passes < MIN_CLASS_COUNT: + continue + if labeled != passes + nonpass or labeled != overall + proxy: + raise PublicSchemaError("Label-source quality totals are inconsistent") + rows.append( + { + "year": int(year), + "source_era": _safe_category(source, "Source era"), + "support_rounded": _round_support(n), + "labeled_rate": _rate(labeled, n), + "overall_result_share": _rate(overall, labeled), + "utah_obd_proxy_share": _rate(proxy, labeled), + } + ) + return rows + + +def _base_asset( + development_preview: bool, + population_estimate_allowed: bool, +) -> Dict[str, object]: + return { + "schema_version": SCHEMA_VERSION, + "development_preview": development_preview, + "population_estimate_allowed": population_estimate_allowed, + } + + +def _release_id( + mart_digest: str, + model_provenance: Sequence[Tuple[str, str]], +) -> str: + material = { + "schema_version": SCHEMA_VERSION, + "mart_sha256": mart_digest, + "models": [ + {"model_version": version, "metrics_sha256": metrics_digest} + for version, metrics_digest in sorted(model_provenance) + ], + } + encoded = json.dumps( + material, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def build_assets( + mart: Path, + mart_manifest_path: Path, + bundles: Sequence[ModelBundle], + development_preview: bool, +) -> Dict[str, object]: + mart_manifest, development_source, mart_digest = validate_mart_manifest( + mart, mart_manifest_path, development_preview + ) + population_allowed = bool( + mart_manifest.get("population_estimate_allowed") + ) and not development_preview and not development_source + + connection = duckdb.connect(":memory:") + try: + _create_safe_mart(connection, mart) + diagnostics, model_provenance = load_model_diagnostics( + bundles, + mart_digest, + _partition_vehicle_support(connection), + ) + overview = _overview_rows(connection) + age_risk = _age_risk_rows(connection) + scorecards = _scorecard_rows(connection) + coverage = _coverage_rows(connection) + finally: + connection.close() + if not overview: + raise PublicSchemaError("No overview cells survive publication suppression") + + base = _base_asset(development_preview, population_allowed) + assets: Dict[str, object] = { + "overview_period_county.json": {**base, "rows": overview}, + "age_risk_curve.json": {**base, "rows": age_risk}, + "cohort_scorecard.json": {**base, "rows": scorecards}, + "model_diagnostics.json": {**base, "rows": diagnostics}, + "coverage_quality.json": {**base, "rows": coverage}, + } + + counties = sorted({str(row["public_county"]) for row in overview}) + periods_by_year: Dict[int, set] = {} + for row in overview: + periods_by_year.setdefault(int(row["year"]), set()).add(int(row["quarter"])) + periods = [ + {"year": year, "quarters": sorted(quarters)} + for year, quarters in sorted(periods_by_year.items()) + ] + make_models = [ + {"prior_make": row["prior_make"], "prior_model": row["prior_model"]} + for row in scorecards + ] + model_names = sorted({str(row["model"]) for row in diagnostics}) + assets["filter_catalog.json"] = { + **base, + "public_counties": counties, + "age_bands": [str(row["age_band"]) for row in age_risk], + "prior_make_models": make_models, + "periods": periods, + "models": model_names, + "partitions": list(ALLOWED_PARTITIONS), + } + + source_years = [int(row["year"]) for row in overview] + model_versions = sorted({version for version, _ in model_provenance}) + assets["data_manifest.json"] = { + **base, + "release_id": _release_id(mart_digest, model_provenance), + "model_versions": model_versions, + "data_scope": { + "first_year": min(source_years), + "last_year": max(source_years), + "partitions": list(ALLOWED_PARTITIONS), + "model_names": model_names, + }, + "definitions": { + "target": "first-attempt next-episode binary non-pass rate", + "episode_gap_days": int(mart_manifest.get("episode_gap_days", 30)), + "support_rounding": SUPPORT_ROUNDING, + "suppression_min_support": MIN_SUPPORT, + "suppression_min_pass": MIN_CLASS_COUNT, + "suppression_min_nonpass": MIN_CLASS_COUNT, + "suppression_min_distinct_vehicles": MIN_SUPPORT, + "suppression_min_distinct_pass_vehicles": MIN_CLASS_COUNT, + "suppression_min_distinct_nonpass_vehicles": MIN_CLASS_COUNT, + "locked_test_metrics_published": False, + }, + "assets": sorted( + name for name in PUBLIC_ASSET_NAMES if name != "data_manifest.json" + ), + } + + for name, asset in assets.items(): + validate_public_asset(name, asset) + return assets + + +def _validate_flags(asset: Mapping[str, object]) -> None: + if asset.get("schema_version") != SCHEMA_VERSION: + raise PublicSchemaError("Asset schema version is invalid") + if not isinstance(asset.get("development_preview"), bool): + raise PublicSchemaError("Preview flag must be boolean") + if not isinstance(asset.get("population_estimate_allowed"), bool): + raise PublicSchemaError("Population flag must be boolean") + if asset["development_preview"] and asset["population_estimate_allowed"]: + raise PublicSchemaError("Preview assets cannot permit population estimates") + + +def _scan_public_value(value: object, path: str = "root") -> None: + if isinstance(value, dict): + for key, child in value.items(): + normalized = key.lower() + if normalized in DENIED_EXACT_KEYS or any( + fragment in normalized for fragment in DENIED_KEY_FRAGMENTS + ): + raise PublicSchemaError("Denied public key at " + path) + _scan_public_value(child, path + "." + key) + elif isinstance(value, list): + for index, child in enumerate(value): + _scan_public_value(child, path + "[{}]".format(index)) + elif isinstance(value, str): + if EXACT_TIMESTAMP_PATTERN.match(value): + raise PublicSchemaError("Exact timestamp in public asset") + elif isinstance(value, float): + if not math.isfinite(value): + raise PublicSchemaError("Non-finite public number") + elif value is not None and not isinstance(value, (bool, int)): + raise PublicSchemaError("Unsupported public JSON value") + + +def _validate_rate(value: object, label: str) -> None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise PublicSchemaError(label + " must be numeric") + number = float(value) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise PublicSchemaError(label + " is outside [0,1]") + + +def validate_public_asset(name: str, asset: object) -> None: + if not isinstance(asset, dict): + raise PublicSchemaError("Public asset must be an object") + _validate_flags(asset) + _scan_public_value(asset) + + if name in ROW_ASSET_SCHEMAS: + if set(asset) != { + "schema_version", + "development_preview", + "population_estimate_allowed", + "rows", + }: + raise PublicSchemaError("Row asset has unexpected top-level keys") + rows = asset.get("rows") + if not isinstance(rows, list): + raise PublicSchemaError("Row asset rows must be an array") + required = set(ROW_ASSET_SCHEMAS[name]) + for row in rows: + if not isinstance(row, dict) or set(row) != required: + raise PublicSchemaError("Row does not match its public schema") + if "support_rounded" in row: + support = row["support_rounded"] + if ( + isinstance(support, bool) + or not isinstance(support, int) + or support < MIN_SUPPORT + or support % SUPPORT_ROUNDING + ): + raise PublicSchemaError("Published support is not safely rounded") + for key, value in row.items(): + if key.endswith("_rate") or key.endswith("_share") or key in { + "average_precision", + "brier", + "roc_auc", + "top_10_capture", + }: + _validate_rate(value, key) + if key == "log_loss": + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or float(value) < 0.0 + ): + raise PublicSchemaError("Invalid public log loss") + return + + if name == "filter_catalog.json": + expected = { + "schema_version", + "development_preview", + "population_estimate_allowed", + "public_counties", + "age_bands", + "prior_make_models", + "periods", + "models", + "partitions", + } + if set(asset) != expected: + raise PublicSchemaError("Filter catalog schema is invalid") + if asset["partitions"] != list(ALLOWED_PARTITIONS): + raise PublicSchemaError("Filter catalog exposes an invalid partition") + return + + if name == "data_manifest.json": + expected = { + "schema_version", + "development_preview", + "population_estimate_allowed", + "release_id", + "model_versions", + "data_scope", + "definitions", + "assets", + } + if set(asset) != expected: + raise PublicSchemaError("Data manifest schema is invalid") + definitions = asset.get("definitions") + if not isinstance(definitions, dict) or definitions.get( + "locked_test_metrics_published" + ) is not False: + raise PublicSchemaError("Data manifest does not lock 2025 metrics") + release_id = asset.get("release_id") + if not isinstance(release_id, str) or not re.fullmatch( + r"[0-9a-f]{64}", release_id + ): + raise PublicSchemaError("Data manifest release ID is invalid") + model_versions = asset.get("model_versions") + if ( + not isinstance(model_versions, list) + or not model_versions + or model_versions != sorted(set(model_versions)) + or any( + not isinstance(version, str) + or not SAFE_MODEL_NAME_PATTERN.fullmatch(version) + for version in model_versions + ) + ): + raise PublicSchemaError("Data manifest model versions are invalid") + return + raise PublicSchemaError("Unexpected public asset name") + + +def _json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _validate_output_dir(output_dir: Path) -> Path: + resolved = output_dir.resolve() + if not resolved.is_relative_to(PUBLIC_DATA_ROOT): + raise PublicSchemaError( + "Dashboard assets must remain under dashboard/public/data" + ) + return resolved + + +def _validate_existing_output_contract( + output_dir: Path, + expected_names: frozenset, + require_complete: bool = False, +) -> Dict[str, Path]: + if not output_dir.exists(): + return {} + if not output_dir.is_dir(): + raise PublicSchemaError("Dashboard data output is not a directory") + + entries = {entry.name: entry for entry in output_dir.iterdir()} + unexpected = sorted(set(entries) - expected_names) + invalid_types = sorted( + name + for name, entry in entries.items() + if entry.is_symlink() or not entry.is_file() + ) + if unexpected or invalid_types: + raise PublicSchemaError( + "Dashboard data directory contains unexpected content" + ) + if require_complete and set(entries) != expected_names: + raise PublicSchemaError( + "Dashboard data directory does not match the exact public contract" + ) + return entries + + +def publish_assets( + assets: Mapping[str, object], + output_dir: Path, + overwrite: bool, +) -> Dict[str, Path]: + output_dir = _validate_output_dir(output_dir) + if set(assets) != set(PUBLIC_ASSET_NAMES): + raise PublicSchemaError("Asset set does not match the public contract") + + serialized = {name: _json_bytes(assets[name]) for name in sorted(assets)} + first_asset = next(iter(assets.values())) + if not isinstance(first_asset, dict): + raise PublicSchemaError("Invalid public asset") + sha_manifest = { + "schema_version": SCHEMA_VERSION, + "development_preview": first_asset["development_preview"], + "population_estimate_allowed": first_asset["population_estimate_allowed"], + "files": [ + { + "name": name, + "sha256": hashlib.sha256(serialized[name]).hexdigest(), + } + for name in sorted(serialized) + ], + } + _validate_flags(sha_manifest) + _scan_public_value(sha_manifest) + serialized[SHA256_MANIFEST_NAME] = _json_bytes(sha_manifest) + + targets = {name: output_dir / name for name in serialized} + if set(targets) != PUBLIC_OUTPUT_NAMES: + raise PublicSchemaError("Output file set does not match the public contract") + existing = _validate_existing_output_contract( + output_dir, PUBLIC_OUTPUT_NAMES + ) + if not overwrite and existing: + raise FileExistsError("Refusing to overwrite existing dashboard assets") + + os.umask(0o022) + output_dir.parent.mkdir(parents=True, exist_ok=True) + staging_root = Path( + tempfile.mkdtemp(prefix=".dashboard-data-", dir=str(output_dir.parent)) + ) + try: + for name, content in serialized.items(): + staged = staging_root / name + with staged.open("xb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + staged.chmod(0o644) + + output_dir.mkdir(parents=True, exist_ok=True) + existing = _validate_existing_output_contract( + output_dir, PUBLIC_OUTPUT_NAMES + ) + if not overwrite and existing: + raise FileExistsError("Dashboard assets appeared during publication") + # Publish the checksum manifest last; consumers can treat it as the + # commit marker for the individually atomic file replacements. + for name in sorted(serialized): + if name == SHA256_MANIFEST_NAME: + continue + os.replace(staging_root / name, targets[name]) + os.replace( + staging_root / SHA256_MANIFEST_NAME, + targets[SHA256_MANIFEST_NAME], + ) + _validate_existing_output_contract( + output_dir, PUBLIC_OUTPUT_NAMES, require_complete=True + ) + finally: + shutil.rmtree(staging_root, ignore_errors=True) + return targets + + +def export_dashboard_data( + mart: Path, + mart_manifest: Path, + bundles: Sequence[ModelBundle], + output_dir: Path, + development_preview: bool = False, + overwrite: bool = False, +) -> Dict[str, Path]: + assets = build_assets( + mart=mart, + mart_manifest_path=mart_manifest, + bundles=bundles, + development_preview=development_preview, + ) + return publish_assets(assets, output_dir, overwrite) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parse_args(argv) + if len(args.model_manifest) != len(args.model_metrics): + print( + "Configuration error: pair each model manifest with one metrics file", + file=sys.stderr, + ) + return 2 + bundles = [ + ModelBundle(manifest=manifest, metrics=metrics) + for manifest, metrics in zip(args.model_manifest, args.model_metrics) + ] + try: + paths = export_dashboard_data( + mart=args.mart, + mart_manifest=_mart_manifest_path(args.mart, args.mart_manifest), + bundles=bundles, + output_dir=args.output_dir, + development_preview=args.development_preview, + overwrite=args.overwrite, + ) + except (DashboardExportError, FileExistsError) as exc: + print("Dashboard export refused: " + str(exc), file=sys.stderr) + return 2 + print("Published {} sanitized dashboard assets".format(len(paths))) + print("Public data directory: " + str(args.output_dir.resolve())) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export_history_sample.py b/scripts/export_history_sample.py new file mode 100644 index 0000000..3fa3a51 --- /dev/null +++ b/scripts/export_history_sample.py @@ -0,0 +1,234 @@ +"""Export complete histories for a private development-only vehicle sample. + +The source sample is page-based and therefore not population-representative. +Use it to develop and test the episode/model pipeline, never for final rates. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import psycopg2 + +try: + from . import export_inspection_batch as secure_export +except ImportError: # Direct execution: python scripts/export_history_sample.py + import export_inspection_batch as secure_export + + +QUERY_VERSION = "inspection_history_development_sample_v3" + + +def parse_timestamp(value: str) -> datetime: + try: + return datetime.fromisoformat(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"Invalid ISO timestamp: {value}") from exc + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vehicles", type=int, default=10_000) + parser.add_argument("--sample-percent", type=float, default=0.5) + parser.add_argument("--seed", type=int, default=20260715) + parser.add_argument("--start", type=parse_timestamp, default=datetime(2010, 1, 1)) + parser.add_argument("--end", type=parse_timestamp, default=datetime(2026, 7, 1)) + parser.add_argument("--fetch-size", type=int, default=5_000) + parser.add_argument("--output", type=Path) + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> Path: + if not 100 <= args.vehicles <= 100_000: + raise ValueError("--vehicles must be between 100 and 100000") + if not 0 < args.sample_percent <= 5: + raise ValueError("--sample-percent must be greater than 0 and at most 5") + if not 100 <= args.fetch_size <= 50_000: + raise ValueError("--fetch-size must be between 100 and 50000") + if args.end <= args.start: + raise ValueError("--end must be later than --start") + + output = args.output + if output is None: + output = ( + secure_export.PRIVATE_DATA_ROOT + / "development" + / f"history_sample_{args.vehicles}.csv.gz" + ) + output = output.resolve() + if not output.is_relative_to(secure_export.PRIVATE_DATA_ROOT): + raise ValueError( + f"Development histories must stay under {secure_export.PRIVATE_DATA_ROOT}" + ) + manifest = output.with_name(output.name + ".manifest.json") + if (output.exists() or manifest.exists()) and not args.overwrite: + raise FileExistsError( + f"Refusing to overwrite {output} or its manifest; use --overwrite" + ) + return output + + +def make_sql(sample_percent: float, seed: int) -> str: + safe_percent = f"{sample_percent:.6f}" + safe_seed = int(seed) + return f""" +WITH candidate_vins AS MATERIALIZED ( + SELECT DISTINCT vin + FROM data.inspection_search + TABLESAMPLE SYSTEM ({safe_percent}) REPEATABLE ({safe_seed}) + WHERE vin IS NOT NULL + AND length(btrim(vin)) = 17 +), +sampled_vins AS MATERIALIZED ( + SELECT vin + FROM candidate_vins + ORDER BY md5(vin) + LIMIT %(vehicle_limit)s +) +SELECT + s.id AS internal_event_id, + s.vin, + s.test_start AS event_ts, + lower(btrim(s.county)) AS source_era, + CASE + WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake' + ELSE lower(btrim(s.county)) + END AS public_county, + {secure_export.OUTCOME_SELECT_SQL}, + lower(nullif(btrim(s.program_type), '')) AS program_type, + upper(nullif(btrim(s.test_type), '')) AS test_type, + upper(nullif(btrim(v.make), '')) AS observed_make, + upper(nullif(btrim(v.model), '')) AS observed_model, + v.year AS observed_model_year +FROM data.inspection_search AS s +JOIN data.inspection_vehicle AS v USING (id) +JOIN sampled_vins ON sampled_vins.vin = s.vin +WHERE s.test_start >= %(start_ts)s + AND s.test_start < %(end_ts)s +ORDER BY s.test_start, s.id +""" + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + args = parse_args() + os.umask(0o077) + try: + output = validate_args(args) + key, key_version, key_fingerprint = secure_export.get_hmac_configuration() + except (FileExistsError, ValueError) as exc: + print(f"Configuration error: {exc}", file=sys.stderr) + return 2 + + output.parent.mkdir(parents=True, exist_ok=True) + manifest = output.with_name(output.name + ".manifest.json") + partial = output.with_name(output.name + ".partial") + partial_manifest = manifest.with_name(manifest.name + ".partial") + sql = make_sql(args.sample_percent, args.seed) + + try: + connection = secure_export.connect_read_only() + except (ValueError, RuntimeError, psycopg2.Error) as exc: + print(f"Secure read-only connection failed: {type(exc).__name__}", file=sys.stderr) + return 1 + + source_rows_read = 0 + rows_written = 0 + invalid_vins_skipped = 0 + last_order_key: Optional[tuple[object, object]] = None + try: + with gzip.open(partial, "wt", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(secure_export.OUTPUT_COLUMNS) + with connection.cursor(name="uvh_history_sample") as cursor: + cursor.itersize = args.fetch_size + cursor.execute( + sql, + { + "vehicle_limit": args.vehicles, + "start_ts": args.start, + "end_ts": args.end, + }, + ) + while True: + rows = cursor.fetchmany(args.fetch_size) + if not rows: + break + for source_row in rows: + source_rows_read += 1 + order_key = (source_row[2], source_row[0]) + if last_order_key is not None and order_key <= last_order_key: + raise RuntimeError( + "Source join did not return a unique increasing " + "event_ts/internal_event_id order" + ) + last_order_key = order_key + transformed = secure_export.private_row( + source_row, key, key_version + ) + if transformed is None: + invalid_vins_skipped += 1 + continue + writer.writerow(transformed) + rows_written += 1 + connection.rollback() + except Exception: + partial.unlink(missing_ok=True) + partial_manifest.unlink(missing_ok=True) + raise + finally: + connection.close() + + manifest_data = { + "export_kind": "vehicle_history_development_sample", + "query_version": QUERY_VERSION, + "query_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "source_start_inclusive": args.start.isoformat(), + "source_end_exclusive": args.end.isoformat(), + "vehicle_sample_limit": args.vehicles, + "sample_method": "page_sample_candidates_then_md5_ordered_distinct_vins", + "sample_percent": args.sample_percent, + "sample_seed": args.seed, + "population_estimate_allowed": False, + "vehicle_token_key_version": key_version, + "vehicle_token_key_fingerprint": key_fingerprint, + "source_rows_read": source_rows_read, + "rows": rows_written, + "invalid_vins_skipped": invalid_vins_skipped, + "columns": secure_export.OUTPUT_COLUMNS, + "compressed_file_sha256": file_sha256(partial), + "compressed_file_bytes": partial.stat().st_size, + "classification": "private_pseudonymized_development_only", + "source_transaction": "read_only_repeatable_read", + } + partial_manifest.write_text( + json.dumps(manifest_data, indent=2) + "\n", encoding="utf-8" + ) + os.replace(partial, output) + os.replace(partial_manifest, manifest) + print(f"Exported {rows_written:,} rows for a development vehicle sample") + print(f"Private output: {output}") + print("This sample is not valid for population-rate estimates.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export_inspection_batch.py b/scripts/export_inspection_batch.py new file mode 100644 index 0000000..844dff7 --- /dev/null +++ b/scripts/export_inspection_batch.py @@ -0,0 +1,414 @@ +"""Export one private inspection batch while replacing VINs with keyed HMACs. + +The output is private analytical staging data, not a public-dashboard dataset. +It intentionally omits plates, ZIPs, stations, raw JSON, and operational data. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import hmac +import json +import os +import re +import stat +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable, Optional, Sequence + +import psycopg2 + + +QUERY_VERSION = "inspection_batch_v4" +MAX_BATCH_DAYS = 32 +SECURE_SSL_MODES = {"require", "verify-ca", "verify-full"} +PROJECT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_KEY_FILE = PROJECT_ROOT / ".secrets/vin_hmac.key" +PRIVATE_DATA_ROOT = (PROJECT_ROOT / "data/private").resolve() +VIN_PATTERN = re.compile(r"^[A-HJ-NPR-Z0-9]{17}$") +KEY_VERSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,32}$") + +OUTCOME_ALIASES = { + "PASS": "pass", + "P": "pass", + "FAIL": "fail", + "F": "fail", + "REJECT": "reject", + "ABORT": "abort", +} + + +def _normalized_outcome(value: object) -> Optional[str]: + if not isinstance(value, str): + return None + return OUTCOME_ALIASES.get(value.strip().upper()) + + +def canonicalize_outcome( + overall_result: object, + obd_result: object, + source_era: object, + program_type: object = None, + test_type: object = None, +) -> tuple[Optional[str], Optional[str]]: + """Return the approved label and its provenance for one inspection.""" + overall = _normalized_outcome(overall_result) + if overall is not None: + return overall, "overall_result" + + proxy_eligible = ( + isinstance(source_era, str) + and source_era.strip().lower() == "utah" + and isinstance(program_type, str) + and program_type.strip().lower() == "obd" + and isinstance(test_type, str) + and test_type.strip().upper() == "OBD" + ) + if proxy_eligible: + proxy = _normalized_outcome(obd_result) + if proxy is not None: + # This narrowly validated Utah OBD/OBD proxy is approved only for + # binary pass/non-pass labels. Multiclass work must retain + # overall_result labels exclusively. + return proxy, "utah_obd_proxy" + return None, None + + +def _outcome_case_sql(column: str) -> str: + cases = " ".join( + f"WHEN '{raw}' THEN '{canonical}'" + for raw, canonical in OUTCOME_ALIASES.items() + ) + return f"CASE upper(btrim({column})) {cases} ELSE NULL END" + + +_OVERALL_OUTCOME_SQL = _outcome_case_sql("s.overall_result") +_UTAH_OBD_OUTCOME_SQL = _outcome_case_sql("s.obd_result") +_UTAH_OBD_PROXY_PREDICATE_SQL = """lower(btrim(s.county)) = 'utah' + AND lower(btrim(s.program_type)) = 'obd' + AND upper(btrim(s.test_type)) = 'OBD'""" + +# Keep label provenance in private staging. Only Utah rows normalized to both +# program_type=obd and test_type=OBD may use the proxy, and then only for binary +# pass/non-pass modeling. Multiclass labels must use overall_result. +OUTCOME_SELECT_SQL = f"""CASE + WHEN {_OVERALL_OUTCOME_SQL} IS NOT NULL THEN {_OVERALL_OUTCOME_SQL} + WHEN {_UTAH_OBD_PROXY_PREDICATE_SQL} THEN {_UTAH_OBD_OUTCOME_SQL} + ELSE NULL + END AS canonical_outcome, + CASE + WHEN {_OVERALL_OUTCOME_SQL} IS NOT NULL THEN 'overall_result' + WHEN {_UTAH_OBD_PROXY_PREDICATE_SQL} + AND {_UTAH_OBD_OUTCOME_SQL} IS NOT NULL THEN 'utah_obd_proxy' + ELSE NULL + END AS outcome_label_source""" + + +EXTRACT_SQL = f""" +SELECT + s.id AS internal_event_id, + s.vin, + s.test_start AS event_ts, + lower(btrim(s.county)) AS source_era, + CASE + WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake' + ELSE lower(btrim(s.county)) + END AS public_county, + {OUTCOME_SELECT_SQL}, + lower(nullif(btrim(s.program_type), '')) AS program_type, + upper(nullif(btrim(s.test_type), '')) AS test_type, + upper(nullif(btrim(v.make), '')) AS observed_make, + upper(nullif(btrim(v.model), '')) AS observed_model, + v.year AS observed_model_year +FROM data.inspection_search AS s +JOIN data.inspection_vehicle AS v USING (id) +WHERE s.test_start >= %(start_ts)s + AND s.test_start < %(end_ts)s + AND s.vin IS NOT NULL + AND length(btrim(s.vin)) = 17 +ORDER BY s.test_start, s.id +""" + +OUTPUT_COLUMNS = ( + "internal_event_id", + "vehicle_token", + "vehicle_bucket", + "event_ts", + "source_era", + "public_county", + "canonical_outcome", + "outcome_label_source", + "program_type", + "test_type", + "observed_make", + "observed_model", + "observed_model_year", +) + + +def parse_timestamp(value: str) -> datetime: + try: + return datetime.fromisoformat(value) + except ValueError as exc: + raise argparse.ArgumentTypeError( + f"Invalid ISO timestamp or date: {value}" + ) from exc + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--start", required=True, type=parse_timestamp) + parser.add_argument("--end", required=True, type=parse_timestamp) + parser.add_argument("--output", type=Path) + parser.add_argument("--page-size", type=int, default=10_000) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument( + "--allow-external-output", + action="store_true", + help="Allow private row-level output outside data/private (unsafe).", + ) + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> Path: + if args.end <= args.start: + raise ValueError("--end must be later than --start") + if (args.end - args.start).total_seconds() > MAX_BATCH_DAYS * 86400: + raise ValueError( + f"A source batch may span at most {MAX_BATCH_DAYS} days; " + "export longer periods as closed monthly batches." + ) + if not 100 <= args.page_size <= 50_000: + raise ValueError("--page-size must be between 100 and 50000") + + output = args.output + if output is None: + name = f"inspection_{args.start:%Y%m%d}_{args.end:%Y%m%d}.csv.gz" + output = PRIVATE_DATA_ROOT / "inspection_batches" / name + output = output.resolve() + if not args.allow_external_output and not output.is_relative_to(PRIVATE_DATA_ROOT): + raise ValueError( + f"Private exports must stay under {PRIVATE_DATA_ROOT}; " + "use --allow-external-output only for controlled validation." + ) + manifest = output.with_name(output.name + ".manifest.json") + if (output.exists() or manifest.exists()) and not args.overwrite: + raise FileExistsError( + f"Refusing to overwrite {output} or its manifest; use --overwrite" + ) + return output + + +def get_hmac_configuration() -> tuple[bytes, str, str]: + encoded_key = os.environ.get("VIN_HASH_KEY", "").strip() + key_file = Path(os.environ.get("VIN_HASH_KEY_FILE", DEFAULT_KEY_FILE)) + if not encoded_key and key_file.exists(): + mode = stat.S_IMODE(key_file.stat().st_mode) + if mode & 0o077: + raise ValueError(f"VIN HMAC key file must be mode 0600: {key_file}") + encoded_key = key_file.read_text(encoding="ascii").strip() + + version = os.environ.get("VIN_HASH_KEY_VERSION", "v1").strip() or "v1" + if not KEY_VERSION_PATTERN.fullmatch(version): + raise ValueError("VIN_HASH_KEY_VERSION contains unsupported characters") + try: + key = bytes.fromhex(encoded_key) + except ValueError as exc: + raise ValueError("VIN_HASH_KEY must be a hexadecimal value") from exc + if len(key) != 32: + raise ValueError( + "VIN_HASH_KEY must encode exactly 32 random bytes (64 hex characters). " + "See .env.example." + ) + if len(set(key)) < 8: + raise ValueError( + "VIN_HASH_KEY has insufficient byte diversity; generate 32 random bytes." + ) + fingerprint = hashlib.sha256( + b"utah-vehicle-health/key-fingerprint/v1\0" + key + ).hexdigest()[:16] + return key, version, fingerprint + + +def normalize_vin(vin: str) -> Optional[str]: + normalized = vin.strip().upper() + if not VIN_PATTERN.fullmatch(normalized): + return None + if len(set(normalized)) < 4: + return None + if normalized in {"12345678901234567", "98765432109876543"}: + return None + return normalized + + +def vehicle_token(vin: str, key: bytes, key_version: str) -> str: + message = ( + f"utah-vehicle-health/{key_version}/vin\0".encode("ascii") + + vin.encode("ascii") + ) + return hmac.new(key, message, hashlib.sha256).hexdigest() + + +def private_row( + row: Sequence[object], key: bytes, key_version: str +) -> Optional[tuple[object, ...]]: + internal_event_id, vin, *remaining = row + if not isinstance(vin, str): + raise ValueError("The source returned a non-text VIN") + normalized = normalize_vin(vin) + if normalized is None: + return None + token = vehicle_token(normalized, key, key_version) + bucket = int(token[:8], 16) % 100 + return (internal_event_id, token, bucket, *remaining) + + +def batches( + connection: "psycopg2.extensions.connection", + start: datetime, + end: datetime, + page_size: int, +) -> Iterable[list[tuple[object, ...]]]: + with connection.cursor(name="uvh_inspection_batch") as cursor: + cursor.itersize = page_size + cursor.execute(EXTRACT_SQL, {"start_ts": start, "end_ts": end}) + while True: + rows = cursor.fetchmany(page_size) + if not rows: + return + yield rows + + +def connect_read_only() -> "psycopg2.extensions.connection": + ssl_mode = os.environ.get("PGSSLMODE", "").lower() + if ssl_mode not in SECURE_SSL_MODES: + raise ValueError("PGSSLMODE must require TLS before exporting data") + + connection = psycopg2.connect( + application_name="utah_vehicle_health_batch_export", + connect_timeout=10, + options=( + "-c default_transaction_read_only=on " + "-c statement_timeout=120000 " + "-c lock_timeout=3000 " + "-c idle_in_transaction_session_timeout=60000" + ), + ) + connection.set_session( + readonly=True, + isolation_level="REPEATABLE READ", + autocommit=False, + ) + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + current_setting('transaction_read_only'), + ssl + FROM pg_stat_ssl + WHERE pid = pg_backend_pid() + """ + ) + read_only, ssl = cursor.fetchone() + if read_only != "on" or not ssl: + connection.close() + raise RuntimeError("The database session is not read-only over TLS") + return connection + + +def main() -> int: + args = parse_args() + os.umask(0o077) + try: + output = validate_args(args) + key, key_version, key_fingerprint = get_hmac_configuration() + except (FileExistsError, ValueError) as exc: + print(f"Configuration error: {exc}", file=sys.stderr) + return 2 + + output.parent.mkdir(parents=True, exist_ok=True) + partial = output.with_name(output.name + ".partial") + manifest = output.with_name(output.name + ".manifest.json") + partial_manifest = manifest.with_name(manifest.name + ".partial") + + try: + connection = connect_read_only() + except (ValueError, RuntimeError, psycopg2.Error) as exc: + print(f"Secure read-only connection failed: {type(exc).__name__}", file=sys.stderr) + return 1 + + source_rows_read = 0 + rows_written = 0 + invalid_vins_skipped = 0 + last_order_key: Optional[tuple[object, object]] = None + try: + with gzip.open(partial, "wt", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(OUTPUT_COLUMNS) + for source_rows in batches( + connection, args.start, args.end, args.page_size + ): + for source_row in source_rows: + source_rows_read += 1 + order_key = (source_row[2], source_row[0]) + if last_order_key is not None and order_key <= last_order_key: + raise RuntimeError( + "Source join did not return a unique increasing " + "event_ts/internal_event_id order" + ) + last_order_key = order_key + transformed = private_row(source_row, key, key_version) + if transformed is None: + invalid_vins_skipped += 1 + continue + writer.writerow(transformed) + rows_written += 1 + connection.rollback() + except Exception: + partial.unlink(missing_ok=True) + partial_manifest.unlink(missing_ok=True) + raise + finally: + connection.close() + + digest = hashlib.sha256() + with partial.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + + manifest_data = { + "export_kind": "bounded_batch", + "query_version": QUERY_VERSION, + "query_sha256": hashlib.sha256(EXTRACT_SQL.encode("utf-8")).hexdigest(), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "source_start_inclusive": args.start.isoformat(), + "source_end_exclusive": args.end.isoformat(), + "vehicle_token_key_version": key_version, + "vehicle_token_key_fingerprint": key_fingerprint, + "source_rows_read": source_rows_read, + "rows": rows_written, + "invalid_vins_skipped": invalid_vins_skipped, + "columns": OUTPUT_COLUMNS, + "compressed_file_sha256": digest.hexdigest(), + "compressed_file_bytes": partial.stat().st_size, + "classification": "private_pseudonymized_analytical_staging", + "source_transaction": "read_only_repeatable_read", + } + partial_manifest.write_text( + json.dumps(manifest_data, indent=2) + "\n", encoding="utf-8" + ) + os.replace(partial, output) + os.replace(partial_manifest, manifest) + print(f"Exported {rows_written:,} private pseudonymized rows to {output}") + if invalid_vins_skipped: + print(f"Skipped {invalid_vins_skipped:,} invalid/placeholder VIN rows") + print(f"Wrote manifest to {manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_feasibility_check.py b/scripts/run_feasibility_check.py new file mode 100644 index 0000000..56d984f --- /dev/null +++ b/scripts/run_feasibility_check.py @@ -0,0 +1,71 @@ +"""Run the project's fixed aggregate-only feasibility query safely. + +This runner accepts no arbitrary SQL. It executes the reviewed query inside the +same TLS-protected, read-only transaction used by the private exporters and +prints only its aggregate result table. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import psycopg2 + +import export_inspection_batch as secure_export + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +QUERY_PATH = PROJECT_ROOT / "sql/10_episode_cohort_feasibility.sql" + + +def reviewed_query() -> str: + sql = QUERY_PATH.read_text(encoding="utf-8") + begin = "BEGIN TRANSACTION READ ONLY;" + rollback = "ROLLBACK;" + if sql.count(begin) != 1 or sql.count(rollback) != 1: + raise RuntimeError("The reviewed feasibility query wrappers changed") + return sql.replace(begin, "", 1).rsplit(rollback, 1)[0].strip() + + +def main() -> int: + try: + connection = secure_export.connect_read_only() + except (ValueError, RuntimeError, psycopg2.Error) as exc: + print( + f"Secure read-only connection failed: {type(exc).__name__}", + file=sys.stderr, + ) + return 1 + + try: + with connection.cursor() as cursor: + cursor.execute(reviewed_query()) + columns = [description.name for description in cursor.description] + rows = cursor.fetchall() + connection.rollback() + except psycopg2.Error as exc: + connection.rollback() + print( + f"Aggregate feasibility query failed: " + f"{exc.diag.message_primary or type(exc).__name__}", + file=sys.stderr, + ) + return 1 + finally: + connection.close() + + widths = [len(column) for column in columns] + rendered = [["" if value is None else str(value) for value in row] for row in rows] + for row in rendered: + widths = [max(width, len(value)) for width, value in zip(widths, row)] + + print(" ".join(value.ljust(width) for value, width in zip(columns, widths))) + print(" ".join("-" * width for width in widths)) + for row in rendered: + print(" ".join(value.ljust(width) for value, width in zip(row, widths))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_outcome_mapping_audit.py b/scripts/run_outcome_mapping_audit.py new file mode 100644 index 0000000..a9ba83b --- /dev/null +++ b/scripts/run_outcome_mapping_audit.py @@ -0,0 +1,65 @@ +"""Run the fixed aggregate-only source outcome-mapping audit.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import psycopg2 + +import export_inspection_batch as secure_export + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +QUERY_PATH = PROJECT_ROOT / "sql/11_outcome_mapping_audit.sql" + + +def reviewed_query() -> str: + sql = QUERY_PATH.read_text(encoding="utf-8") + begin = "BEGIN TRANSACTION READ ONLY;" + rollback = "ROLLBACK;" + if sql.count(begin) != 1 or sql.count(rollback) != 1: + raise RuntimeError("The reviewed outcome audit wrappers changed") + return sql.replace(begin, "", 1).rsplit(rollback, 1)[0].strip() + + +def main() -> int: + try: + connection = secure_export.connect_read_only() + except (ValueError, RuntimeError, psycopg2.Error) as exc: + print( + f"Secure read-only connection failed: {type(exc).__name__}", + file=sys.stderr, + ) + return 1 + + try: + with connection.cursor() as cursor: + cursor.execute(reviewed_query()) + columns = [description.name for description in cursor.description] + rows = cursor.fetchall() + connection.rollback() + except psycopg2.Error as exc: + connection.rollback() + print( + "Aggregate outcome audit failed: " + f"{exc.diag.message_primary or type(exc).__name__}", + file=sys.stderr, + ) + return 1 + finally: + connection.close() + + widths = [len(column) for column in columns] + rendered = [["" if value is None else str(value) for value in row] for row in rows] + for row in rendered: + widths = [max(width, len(value)) for width, value in zip(widths, row)] + print(" ".join(value.ljust(width) for value, width in zip(columns, widths))) + print(" ".join("-" * width for width in widths)) + for row in rendered: + print(" ".join(value.ljust(width) for value, width in zip(row, widths))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/train_baselines.py b/scripts/train_baselines.py new file mode 100644 index 0000000..922cee0 --- /dev/null +++ b/scripts/train_baselines.py @@ -0,0 +1,1469 @@ +"""Train leakage-safe baselines from the private inspection episode mart. + +This script deliberately accepts only an explicit, prior-only feature schema. +It never queries countydata, never writes row-level predictions, and writes all +model artifacts beneath ``artifacts/private`` or ``data/private``. + +The locked 2025 partition is not evaluated unless ``--evaluate-locked`` is +provided explicitly. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import json +import math +import os +import statistics +import sys +import warnings +from collections import Counter +from dataclasses import dataclass +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Dict, Iterator, List, Mapping, Optional, Sequence, Tuple + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MODEL_VERSION = "baseline_v1" + +ID_COLUMNS = ( + "vehicle_token", + "vehicle_bucket", + "is_vin_audit", + "episode_number", +) +TARGET_COLUMNS = ( + "episode_start", + "first_outcome", + "target_nonpass", + "eligible_returning_target", + "temporal_partition", +) +NUMERIC_FEATURES = ( + "vehicle_age", + "prior_episode_count", + "prior_total_attempt_count", + "prior_attempt_count", + "days_since_prior_episode", + "days_since_prior_adverse", + "prior_nonpass_rate", +) +CATEGORICAL_FEATURES = ( + "public_county", + "target_season", + "prior_first_outcome", + "prior_final_outcome", + "last_observed_make", + "last_observed_model", +) +AUDIT_COLUMNS = ("source_era", "target_outcome_label_source") +REQUIRED_COLUMNS = ( + ID_COLUMNS + TARGET_COLUMNS + NUMERIC_FEATURES + CATEGORICAL_FEATURES + AUDIT_COLUMNS +) + +PASS_OUTCOMES = {"pass", "p"} +NONPASS_OUTCOMES = {"fail", "f", "reject", "abort"} +MISSING_TEXT = {"", "null", "none", "na", "n/a", ""} +ALLOWED_TARGET_LABEL_SOURCES = {"overall_result", "utah_obd_proxy"} + +# This trainer is intentionally binary-only. The Utah OBD proxy establishes a +# pass/non-pass label but is not approved to synthesize pass/fail/reject/abort +# classes. Any future four-class trainer must fail closed on utah_obd_proxy. +TARGET_CONTRACT = { + "task": "binary_pass_vs_nonpass", + "allowed_label_sources": sorted(ALLOWED_TARGET_LABEL_SOURCES), + "utah_obd_proxy_binary_approved": True, + "utah_obd_proxy_four_class_approved": False, +} +PLATT_CALIBRATION_CONFIG = { + "method": "sigmoid_platt_scaling", + "solver": "liblinear", + "penalty": "l2", + "c": 1000.0, +} + +# These fields are forbidden even when they are not selected as model features. +# Their presence indicates that the input is not the approved episode mart. +FORBIDDEN_EXACT_COLUMNS = { + "vin", + "plate", + "zip", + "zipcode", + "station", + "station_id", + "internal_event_id", + "overall_result", + "obd_result", + "result_reason", + "dtc_count", + "mil", + "readiness", + "raw_json", + "eventual_outcome", + "eventually_passed", + "target_attempt_count", + "target_final_outcome", +} +FORBIDDEN_COLUMN_FRAGMENTS = ( + "license_plate", + "certificate", + "calibration_id", + "visual_check", + "readiness_monitor", + "target_dtc", + "target_pid", + "target_obd", + "target_mil", +) +FORBIDDEN_UNKNOWN_COLUMN_TOKENS = { + "address", + "certificate", + "dtc", + "email", + "ip", + "mil", + "obd", + "owner", + "pid", + "plate", + "readiness", + "session", + "station", + "user", + "vin", + "zip", +} + +SPLIT_BOUNDS = { + "train": (date(2016, 1, 1), date(2023, 1, 1)), + "tune": (date(2023, 1, 1), date(2024, 1, 1)), + "calibrate": (date(2024, 1, 1), date(2025, 1, 1)), + "locked_test": (date(2025, 1, 1), date(2026, 1, 1)), + "shadow": (date(2026, 1, 1), date(2027, 1, 1)), +} +MART_PARTITION_NAMES = { + "train": {"train"}, + "tune": {"tune", "validation"}, + "calibrate": {"calibrate", "calibration"}, + "locked_test": {"test", "locked_test", "locked-test"}, + "shadow": {"shadow", "drift", "monitor"}, +} + + +class BaselineError(Exception): + """Base class for expected, user-facing baseline errors.""" + + +class SchemaError(BaselineError): + """Raised when the private mart schema is unsafe or incomplete.""" + + +class DataValidationError(BaselineError): + """Raised when a mart row violates the modeling contract.""" + + +class DependencyError(BaselineError): + """Raised when optional modeling dependencies are unavailable.""" + + +@dataclass(frozen=True) +class EpisodeRow: + """One eligible returning-vehicle target episode.""" + + vehicle_token: str + vehicle_bucket: int + is_vin_audit: bool + episode_number: int + episode_start: datetime + partition: str + target: int + source_era: Optional[str] + target_outcome_label_source: str + prior_first_outcome: Optional[str] + numeric: Mapping[str, Optional[float]] + categorical: Mapping[str, Optional[str]] + + @property + def audit_vehicle(self) -> bool: + return self.is_vin_audit + + +@dataclass +class MartData: + """Validated rows and non-sensitive cohort counts.""" + + rows_by_partition: Dict[str, List[EpisodeRow]] + input_rows: int + ineligible_rows: int + eligible_rows: int + + +@dataclass +class FeatureEncoder: + """Train-only numeric imputation/scaling and categorical rare grouping.""" + + min_category_count: int + numeric_medians: Dict[str, float] + numeric_means: Dict[str, float] + numeric_scales: Dict[str, float] + kept_categories: Dict[str, set] + vectorizer: object + + @classmethod + def fit_transform( + cls, + rows: Sequence[EpisodeRow], + min_category_count: int, + np_module: object, + dict_vectorizer_class: object, + ) -> Tuple["FeatureEncoder", object]: + if not rows: + raise DataValidationError("Cannot fit preprocessing on zero rows") + if min_category_count < 1: + raise DataValidationError("min_category_count must be at least 1") + + numeric_medians: Dict[str, float] = {} + numeric_means: Dict[str, float] = {} + numeric_scales: Dict[str, float] = {} + for name in NUMERIC_FEATURES: + observed = [row.numeric[name] for row in rows if row.numeric[name] is not None] + if not observed: + raise DataValidationError( + "Training data has no observed values for numeric feature " + name + ) + median = float(statistics.median(observed)) + imputed = [median if row.numeric[name] is None else row.numeric[name] for row in rows] + mean = float(sum(imputed) / len(imputed)) + variance = float(sum((value - mean) ** 2 for value in imputed) / len(imputed)) + numeric_medians[name] = median + numeric_means[name] = mean + numeric_scales[name] = math.sqrt(variance) if variance > 0.0 else 1.0 + + kept_categories: Dict[str, set] = {} + for name in CATEGORICAL_FEATURES: + counts = Counter(_category_value(row.categorical[name]) for row in rows) + kept_categories[name] = { + value for value, count in counts.items() if count >= min_category_count + } + + vectorizer = dict_vectorizer_class(dtype=np_module.float64, sparse=True, sort=True) + encoder = cls( + min_category_count=min_category_count, + numeric_medians=numeric_medians, + numeric_means=numeric_means, + numeric_scales=numeric_scales, + kept_categories=kept_categories, + vectorizer=vectorizer, + ) + encoded = vectorizer.fit_transform([encoder.feature_dict(row) for row in rows]) + return encoder, encoded + + def feature_dict(self, row: EpisodeRow) -> Dict[str, object]: + values: Dict[str, object] = {} + for name in NUMERIC_FEATURES: + raw = row.numeric[name] + missing = raw is None + imputed = self.numeric_medians[name] if missing else raw + values["num__" + name] = ( + float(imputed) - self.numeric_means[name] + ) / self.numeric_scales[name] + values["missing__" + name] = 1.0 if missing else 0.0 + + for name in CATEGORICAL_FEATURES: + category = _category_value(row.categorical[name]) + if category not in self.kept_categories[name]: + category = "" + values["cat__" + name] = category + return values + + def transform(self, rows: Sequence[EpisodeRow]) -> object: + return self.vectorizer.transform([self.feature_dict(row) for row in rows]) + + def artifact_state(self) -> Dict[str, object]: + return { + "min_category_count": self.min_category_count, + "numeric_medians": self.numeric_medians, + "numeric_means": self.numeric_means, + "numeric_scales": self.numeric_scales, + "kept_categories": { + name: sorted(values) for name, values in self.kept_categories.items() + }, + "vectorizer": self.vectorizer, + } + + +@dataclass +class TrainedBaselines: + training_prevalence: float + encoder: FeatureEncoder + logistic_model: object + platt_model: object + selected_c: float + selected_n_iter: int + platt_n_iter: int + max_iter: int + tuning_results: List[Dict[str, object]] + + +def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mart", + type=Path, + required=True, + help="Private episode mart (.csv, .csv.gz, or .parquet)", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=PROJECT_ROOT / "artifacts/private/baselines" / MODEL_VERSION, + ) + parser.add_argument("--seed", type=int, default=20260715) + parser.add_argument("--min-category-count", type=int, default=100) + parser.add_argument("--calibration-bins", type=int, default=10) + parser.add_argument("--c-grid", default="0.03,0.1,0.3,1.0,3.0") + parser.add_argument("--max-iter", type=int, default=5000) + parser.add_argument( + "--evaluate-locked", + action="store_true", + help="Explicitly unlock final metric calculation on the 2025 partition", + ) + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args(argv) + + +def parse_c_grid(value: str) -> List[float]: + try: + result = [float(part.strip()) for part in value.split(",") if part.strip()] + except ValueError as exc: + raise DataValidationError("--c-grid must contain positive numbers") from exc + if not result or any(not math.isfinite(item) or item <= 0.0 for item in result): + raise DataValidationError("--c-grid must contain positive finite numbers") + return result + + +def _category_value(value: Optional[str]) -> str: + return "" if value is None else value + + +def _normalize_column(name: str) -> str: + return name.strip().lower() + + +def validate_schema(columns: Sequence[str]) -> None: + normalized = [_normalize_column(name) for name in columns] + if any(not name for name in normalized): + raise SchemaError("The mart contains a blank column name") + if len(normalized) != len(set(normalized)): + raise SchemaError("The mart contains duplicate column names") + + missing = sorted(set(REQUIRED_COLUMNS) - set(normalized)) + if missing: + raise SchemaError("The mart is missing required columns: " + ", ".join(missing)) + + forbidden = [] + for name in normalized: + if name in FORBIDDEN_EXACT_COLUMNS or any( + fragment in name for fragment in FORBIDDEN_COLUMN_FRAGMENTS + ): + forbidden.append(name) + continue + if name not in REQUIRED_COLUMNS and ( + set(name.split("_")) & FORBIDDEN_UNKNOWN_COLUMN_TOKENS + ): + forbidden.append(name) + if forbidden: + raise SchemaError( + "The input contains forbidden identifier/leakage columns: " + + ", ".join(sorted(forbidden)) + ) + + +def _parse_bool(value: object, column: str, row_number: int) -> bool: + normalized = str(value).strip().lower() + if normalized in {"true", "t", "1", "yes", "y"}: + return True + if normalized in {"false", "f", "0", "no", "n"}: + return False + raise DataValidationError( + "Row {} has invalid boolean in {}: {!r}".format(row_number, column, value) + ) + + +def _parse_int(value: object, column: str, row_number: int) -> int: + try: + parsed = int(str(value).strip()) + except (TypeError, ValueError) as exc: + raise DataValidationError( + "Row {} has invalid integer in {}: {!r}".format(row_number, column, value) + ) from exc + return parsed + + +def _parse_optional_float(value: object, column: str, row_number: int) -> Optional[float]: + if value is None or str(value).strip().lower() in MISSING_TEXT: + return None + try: + parsed = float(str(value).strip()) + except (TypeError, ValueError) as exc: + raise DataValidationError( + "Row {} has invalid numeric value in {}: {!r}".format( + row_number, column, value + ) + ) from exc + if not math.isfinite(parsed): + raise DataValidationError( + "Row {} has a non-finite value in {}".format(row_number, column) + ) + return parsed + + +def _parse_optional_category(value: object) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip().lower() + return None if normalized in MISSING_TEXT else normalized + + +def _parse_timestamp(value: object, row_number: int) -> datetime: + text = str(value).strip() + if not text: + raise DataValidationError("Row {} has a blank episode_start".format(row_number)) + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + return datetime.fromisoformat(text) + except ValueError as exc: + raise DataValidationError( + "Row {} has invalid episode_start: {!r}".format(row_number, value) + ) from exc + + +def _binary_outcome(value: object, column: str, row_number: int) -> int: + normalized = str(value).strip().lower() + if normalized in PASS_OUTCOMES: + return 0 + if normalized in NONPASS_OUTCOMES: + return 1 + raise DataValidationError( + "Row {} has unrecognized {}: {!r}".format(row_number, column, value) + ) + + +def _target_nonpass(value: object, row_number: int) -> int: + parsed = _parse_int(value, "target_nonpass", row_number) + if parsed not in (0, 1): + raise DataValidationError( + "Row {} target_nonpass must be 0 or 1".format(row_number) + ) + return parsed + + +def partition_for_timestamp(timestamp: datetime) -> Optional[str]: + target_date = timestamp.date() + for name, (start, end) in SPLIT_BOUNDS.items(): + if start <= target_date < end: + return name + return None + + +def _validate_declared_partition( + declared: object, derived: str, row_number: int +) -> None: + value = str(declared).strip().lower() + if value not in MART_PARTITION_NAMES[derived]: + raise DataValidationError( + "Row {} declares temporal_partition={!r}, but episode_start belongs to {}".format( + row_number, declared, derived + ) + ) + + +def _normalize_record(record: Mapping[str, object]) -> Dict[str, object]: + return {_normalize_column(str(key)): value for key, value in record.items()} + + +def episode_from_record(record: Mapping[str, object], row_number: int) -> Optional[EpisodeRow]: + row = _normalize_record(record) + eligible = _parse_bool( + row["eligible_returning_target"], "eligible_returning_target", row_number + ) + if not eligible: + return None + + token = str(row["vehicle_token"]).strip().lower() + if len(token) != 64: + raise DataValidationError( + "Row {} vehicle_token is not a 64-character HMAC token".format(row_number) + ) + try: + int(token, 16) + except ValueError as exc: + raise DataValidationError( + "Row {} vehicle_token is not hexadecimal".format(row_number) + ) from exc + + bucket = _parse_int(row["vehicle_bucket"], "vehicle_bucket", row_number) + if not 0 <= bucket <= 99: + raise DataValidationError( + "Row {} vehicle_bucket must be between 0 and 99".format(row_number) + ) + is_vin_audit = _parse_bool(row["is_vin_audit"], "is_vin_audit", row_number) + if is_vin_audit != (bucket < 10): + raise DataValidationError( + "Row {} is_vin_audit disagrees with the mart's bucket<10 contract".format( + row_number + ) + ) + + episode_number = _parse_int(row["episode_number"], "episode_number", row_number) + if episode_number <= 1: + raise DataValidationError( + "Row {} is marked eligible but is not a returning episode".format(row_number) + ) + + episode_start = _parse_timestamp(row["episode_start"], row_number) + partition = partition_for_timestamp(episode_start) + if partition is None or partition == "shadow": + # Historical context and post-cutoff rows are not supervised here. Shadow + # monitoring is intentionally outside this locked baseline trainer. + return None + _validate_declared_partition(row["temporal_partition"], partition, row_number) + + target = _binary_outcome(row["first_outcome"], "first_outcome", row_number) + provided_target = _target_nonpass(row["target_nonpass"], row_number) + if target != provided_target: + raise DataValidationError( + "Row {} target_nonpass disagrees with first_outcome".format(row_number) + ) + + numeric = { + name: _parse_optional_float(row[name], name, row_number) + for name in NUMERIC_FEATURES + } + prior_episode_count = numeric["prior_episode_count"] + if prior_episode_count is None or prior_episode_count < 1.0: + raise DataValidationError( + "Row {} is marked eligible but prior_episode_count is below 1".format( + row_number + ) + ) + rate = numeric["prior_nonpass_rate"] + if rate is not None and not 0.0 <= rate <= 1.0: + raise DataValidationError( + "Row {} prior_nonpass_rate is outside [0, 1]".format(row_number) + ) + for name in ( + "prior_episode_count", + "prior_total_attempt_count", + "prior_attempt_count", + "days_since_prior_episode", + "days_since_prior_adverse", + ): + value = numeric[name] + if value is not None and value < 0.0: + raise DataValidationError( + "Row {} {} is negative".format(row_number, name) + ) + + categorical = { + name: _parse_optional_category(row[name]) for name in CATEGORICAL_FEATURES + } + source_era = _parse_optional_category(row["source_era"]) + if source_era is None: + raise DataValidationError( + "Row {} is missing source_era required for drift auditing".format(row_number) + ) + label_source = _parse_optional_category(row["target_outcome_label_source"]) + if label_source not in ALLOWED_TARGET_LABEL_SOURCES: + raise DataValidationError( + "Row {} has unapproved target_outcome_label_source: {!r}".format( + row_number, row["target_outcome_label_source"] + ) + ) + if label_source == "utah_obd_proxy" and source_era != "utah": + raise DataValidationError( + "Row {} uses utah_obd_proxy outside source_era='utah'".format(row_number) + ) + return EpisodeRow( + vehicle_token=token, + vehicle_bucket=bucket, + is_vin_audit=is_vin_audit, + episode_number=episode_number, + episode_start=episode_start, + partition=partition, + target=target, + source_era=source_era, + target_outcome_label_source=label_source, + prior_first_outcome=categorical["prior_first_outcome"], + numeric=numeric, + categorical=categorical, + ) + + +def _csv_records(path: Path) -> Tuple[List[str], Iterator[Mapping[str, object]]]: + opener = gzip.open if path.name.lower().endswith(".gz") else open + with opener(path, "rt", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + if reader.fieldnames is None: + raise SchemaError("The mart has no header row") + columns = list(reader.fieldnames) + + def iterator() -> Iterator[Mapping[str, object]]: + with opener(path, "rt", encoding="utf-8", newline="") as handle: + row_reader = csv.DictReader(handle) + if row_reader.fieldnames is None: # pragma: no cover - file changed mid-run + raise SchemaError("The mart header disappeared while reading") + if list(row_reader.fieldnames) != columns: + raise SchemaError("The mart header changed while reading") + for record in row_reader: + if None in record: + raise SchemaError("A CSV mart row has more fields than its header") + yield record + + return columns, iterator() + + +def _parquet_records(path: Path) -> Tuple[List[str], Iterator[Mapping[str, object]]]: + try: + import duckdb + except ImportError as exc: + raise DependencyError( + "Reading Parquet requires duckdb; use the CSV/CSV.GZ mart or install duckdb" + ) from exc + connection = duckdb.connect(database=":memory:") + try: + cursor = connection.execute("SELECT * FROM read_parquet(?)", [str(path)]) + columns = [description[0] for description in cursor.description] + except Exception as exc: + connection.close() + raise SchemaError("DuckDB could not read the Parquet mart") from exc + connection.close() + + def iterator() -> Iterator[Mapping[str, object]]: + row_connection = duckdb.connect(database=":memory:") + try: + row_cursor = row_connection.execute( + "SELECT * FROM read_parquet(?)", [str(path)] + ) + while True: + rows = row_cursor.fetchmany(65_536) + if not rows: + return + for values in rows: + yield dict(zip(columns, values)) + except Exception as exc: + raise DataValidationError("DuckDB failed while scanning the Parquet mart") from exc + finally: + row_connection.close() + + return columns, iterator() + + +def mart_records(path: Path) -> Tuple[List[str], Iterator[Mapping[str, object]]]: + name = path.name.lower() + if name.endswith(".parquet"): + return _parquet_records(path) + if name.endswith(".csv") or name.endswith(".csv.gz"): + return _csv_records(path) + raise SchemaError("The mart must be .csv, .csv.gz, or .parquet") + + +def load_mart(path: Path) -> MartData: + if not path.is_file(): + raise SchemaError("Mart file does not exist: " + str(path)) + columns, records = mart_records(path) + validate_schema(columns) + + rows_by_partition = {name: [] for name in ("train", "tune", "calibrate", "locked_test")} + input_rows = 0 + ineligible_rows = 0 + for row_number, record in enumerate(records, start=2): + input_rows += 1 + episode = episode_from_record(record, row_number) + if episode is None: + ineligible_rows += 1 + continue + rows_by_partition[episode.partition].append(episode) + + eligible_rows = sum(len(rows) for rows in rows_by_partition.values()) + if eligible_rows == 0: + raise DataValidationError("The mart contains no eligible supervised episodes") + return MartData( + rows_by_partition=rows_by_partition, + input_rows=input_rows, + ineligible_rows=ineligible_rows, + eligible_rows=eligible_rows, + ) + + +def _require_two_classes(rows: Sequence[EpisodeRow], purpose: str) -> None: + classes = {row.target for row in rows} + if classes != {0, 1}: + raise DataValidationError( + "{} requires both pass and non-pass rows; observed classes={}".format( + purpose, sorted(classes) + ) + ) + + +def _non_audit(rows: Sequence[EpisodeRow]) -> List[EpisodeRow]: + return [row for row in rows if not row.audit_vehicle] + + +def _targets(rows: Sequence[EpisodeRow]) -> List[int]: + return [row.target for row in rows] + + +def require_ml_dependencies() -> Tuple[object, object, object, object, object]: + try: + import joblib + import numpy as np + import sklearn + from sklearn.exceptions import ConvergenceWarning + from sklearn.feature_extraction import DictVectorizer + from sklearn.linear_model import LogisticRegression + except ImportError as exc: + raise DependencyError( + "Training requires numpy, scikit-learn, and joblib in the active environment" + ) from exc + return np, sklearn, DictVectorizer, LogisticRegression, ConvergenceWarning + + +def fit_with_convergence_capture( + model: object, + features: object, + targets: object, + convergence_warning_class: object, +) -> List[str]: + """Fit while capturing only convergence warnings and replaying all others.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", convergence_warning_class) + model.fit(features, targets) + + convergence_messages: List[str] = [] + for warning in caught: + if issubclass(warning.category, convergence_warning_class): + convergence_messages.append(str(warning.message)) + continue + # record=True redirects every warning that passed the caller's active + # filters. Replay unrelated warnings rather than silently swallowing + # them as a side effect of convergence auditing. + warnings.warn_explicit( + message=warning.message, + category=warning.category, + filename=warning.filename, + lineno=warning.lineno, + ) + return convergence_messages + + +def model_n_iter(model: object, np_module: object) -> Optional[int]: + values = getattr(model, "n_iter_", None) + if values is None: + return None + array = np_module.asarray(values) + if array.size == 0 or not np_module.isfinite(array).all(): + return None + return int(array.max()) + + +def model_parameters_finite(model: object, np_module: object) -> bool: + coefficient = getattr(model, "coef_", None) + intercept = getattr(model, "intercept_", None) + if coefficient is None or intercept is None: + return False + return bool( + np_module.isfinite(np_module.asarray(coefficient)).all() + and np_module.isfinite(np_module.asarray(intercept)).all() + ) + + +def probability_array_finite(probabilities: object, np_module: object) -> bool: + values = np_module.asarray(probabilities) + return bool( + values.size > 0 + and np_module.isfinite(values).all() + and (values >= 0.0).all() + and (values <= 1.0).all() + ) + + +def train_baselines( + mart: MartData, + c_grid: Sequence[float], + min_category_count: int, + seed: int, + max_iter: int, +) -> TrainedBaselines: + ( + np, + _sklearn, + dict_vectorizer_class, + logistic_class, + convergence_warning_class, + ) = require_ml_dependencies() + train_rows = _non_audit(mart.rows_by_partition["train"]) + tune_rows = _non_audit(mart.rows_by_partition["tune"]) + calibrate_rows = _non_audit(mart.rows_by_partition["calibrate"]) + _require_two_classes(train_rows, "Training") + _require_two_classes(tune_rows, "Tuning") + _require_two_classes(calibrate_rows, "Calibration") + + train_targets = np.asarray(_targets(train_rows), dtype=np.int8) + tune_targets = _targets(tune_rows) + calibrate_targets = np.asarray(_targets(calibrate_rows), dtype=np.int8) + training_prevalence = float(train_targets.mean()) + + encoder, x_train = FeatureEncoder.fit_transform( + train_rows, + min_category_count=min_category_count, + np_module=np, + dict_vectorizer_class=dict_vectorizer_class, + ) + x_tune = encoder.transform(tune_rows) + + candidates: List[Tuple[float, float, float, object, int]] = [] + tuning_results: List[Dict[str, object]] = [] + for c_value in c_grid: + model = logistic_class( + C=float(c_value), + penalty="l2", + solver="saga", + class_weight=None, + max_iter=max_iter, + random_state=seed, + ) + convergence_messages = fit_with_convergence_capture( + model, + x_train, + train_targets, + convergence_warning_class, + ) + n_iter = model_n_iter(model, np) + coefficients_finite = model_parameters_finite(model, np) + probability_values = None + probabilities_finite = False + if coefficients_finite: + probability_values = model.predict_proba(x_tune)[:, 1] + probabilities_finite = probability_array_finite(probability_values, np) + + converged = ( + not convergence_messages + and n_iter is not None + and n_iter < max_iter + ) + eligible_for_selection = ( + converged and coefficients_finite and probabilities_finite + ) + result: Dict[str, object] = { + "c": float(c_value), + "converged": converged, + "n_iter": n_iter, + "convergence_warning": ( + " | ".join(convergence_messages) if convergence_messages else None + ), + "finite_coefficients": coefficients_finite, + "finite_probabilities": probabilities_finite, + "eligible_for_selection": eligible_for_selection, + "brier": None, + "average_precision": None, + } + if eligible_for_selection and probability_values is not None: + probabilities = probability_values.tolist() + metrics = binary_metrics(tune_targets, probabilities) + brier = float(metrics["brier"]) + average_precision = float(metrics["average_precision"]) + result["brier"] = brier + result["average_precision"] = average_precision + candidates.append( + (brier, -average_precision, float(c_value), model, int(n_iter)) + ) + tuning_results.append(result) + + if not candidates: + raise DataValidationError( + "No logistic candidate converged with finite coefficients and " + "probabilities; increase --max-iter or inspect feature scaling" + ) + + _best_brier, _best_negative_ap, selected_c, selected_model, selected_n_iter = min( + candidates, key=lambda item: (item[0], item[1], item[2]) + ) + if not model_parameters_finite(selected_model, np): + raise DataValidationError("Selected logistic model has non-finite coefficients") + selected_tune_probabilities = selected_model.predict_proba(x_tune)[:, 1] + if not probability_array_finite(selected_tune_probabilities, np): + raise DataValidationError("Selected logistic model has non-finite probabilities") + + # Platt scaling is fitted only on the dedicated 2024 calibration partition. + x_calibrate = encoder.transform(calibrate_rows) + decision_scores = selected_model.decision_function(x_calibrate).reshape(-1, 1) + if not bool(np.isfinite(decision_scores).all()): + raise DataValidationError("Selected logistic model has non-finite decision scores") + # C=1000 with liblinear is a deliberately finite approximation to an + # unregularized sigmoid calibrator. It is stable for this Python 3.9 / + # scikit-learn 1.6 / NumPy 2.0 stack, where lbfgs emitted numeric warnings. + platt_model = logistic_class( + C=PLATT_CALIBRATION_CONFIG["c"], + penalty=PLATT_CALIBRATION_CONFIG["penalty"], + solver=PLATT_CALIBRATION_CONFIG["solver"], + class_weight=None, + max_iter=max_iter, + random_state=seed, + ) + platt_warnings = fit_with_convergence_capture( + platt_model, + decision_scores, + calibrate_targets, + convergence_warning_class, + ) + platt_n_iter = model_n_iter(platt_model, np) + if platt_warnings or platt_n_iter is None or platt_n_iter >= max_iter: + if platt_warnings: + detail = " | ".join(platt_warnings) + elif platt_n_iter is None: + detail = "missing n_iter_" + else: + detail = "n_iter reached max_iter without a ConvergenceWarning" + raise DataValidationError("Platt calibration did not converge: " + detail) + if not model_parameters_finite(platt_model, np): + raise DataValidationError("Platt calibration has non-finite coefficients") + calibrated_probabilities = platt_model.predict_proba(decision_scores)[:, 1] + if not probability_array_finite(calibrated_probabilities, np): + raise DataValidationError("Platt calibration has non-finite probabilities") + + return TrainedBaselines( + training_prevalence=training_prevalence, + encoder=encoder, + logistic_model=selected_model, + platt_model=platt_model, + selected_c=selected_c, + selected_n_iter=selected_n_iter, + platt_n_iter=platt_n_iter, + max_iter=max_iter, + tuning_results=tuning_results, + ) + + +def literal_previous_probabilities( + rows: Sequence[EpisodeRow], training_prevalence: float +) -> Tuple[List[float], int]: + probabilities: List[float] = [] + fallback_count = 0 + for row in rows: + outcome = row.prior_first_outcome + if outcome in PASS_OUTCOMES: + probabilities.append(0.0) + elif outcome in NONPASS_OUTCOMES: + probabilities.append(1.0) + else: + probabilities.append(training_prevalence) + fallback_count += 1 + return probabilities, fallback_count + + +def logistic_probabilities( + trained: TrainedBaselines, rows: Sequence[EpisodeRow], calibrated: bool +) -> List[float]: + matrix = trained.encoder.transform(rows) + if calibrated: + decision = trained.logistic_model.decision_function(matrix).reshape(-1, 1) + return trained.platt_model.predict_proba(decision)[:, 1].tolist() + return trained.logistic_model.predict_proba(matrix)[:, 1].tolist() + + +def _validate_probabilities(targets: Sequence[int], probabilities: Sequence[float]) -> None: + if len(targets) != len(probabilities) or not targets: + raise DataValidationError("Targets and probabilities must have equal nonzero length") + if any(target not in (0, 1) for target in targets): + raise DataValidationError("Metrics require binary targets") + for probability in probabilities: + if not math.isfinite(probability) or not 0.0 <= probability <= 1.0: + raise DataValidationError("Predicted probabilities must be finite and in [0, 1]") + + +def average_precision(targets: Sequence[int], probabilities: Sequence[float]) -> Optional[float]: + positives = sum(targets) + if positives == 0: + return None + ordered = sorted(zip(probabilities, targets), key=lambda item: item[0], reverse=True) + true_positives = 0 + false_positives = 0 + previous_recall = 0.0 + result = 0.0 + index = 0 + while index < len(ordered): + score = ordered[index][0] + group_positive = 0 + group_total = 0 + while index < len(ordered) and ordered[index][0] == score: + group_positive += ordered[index][1] + group_total += 1 + index += 1 + true_positives += group_positive + false_positives += group_total - group_positive + recall = true_positives / positives + precision = true_positives / (true_positives + false_positives) + result += (recall - previous_recall) * precision + previous_recall = recall + return result + + +def roc_auc(targets: Sequence[int], probabilities: Sequence[float]) -> Optional[float]: + positives = sum(targets) + negatives = len(targets) - positives + if positives == 0 or negatives == 0: + return None + ordered = sorted(zip(probabilities, targets), key=lambda item: item[0], reverse=True) + true_positives = 0 + false_positives = 0 + previous_tpr = 0.0 + previous_fpr = 0.0 + area = 0.0 + index = 0 + while index < len(ordered): + score = ordered[index][0] + group_positive = 0 + group_total = 0 + while index < len(ordered) and ordered[index][0] == score: + group_positive += ordered[index][1] + group_total += 1 + index += 1 + true_positives += group_positive + false_positives += group_total - group_positive + tpr = true_positives / positives + fpr = false_positives / negatives + area += (fpr - previous_fpr) * (tpr + previous_tpr) / 2.0 + previous_tpr = tpr + previous_fpr = fpr + return area + + +def top_capacity_metrics( + targets: Sequence[int], probabilities: Sequence[float], capacity: float +) -> Dict[str, Optional[float]]: + if not 0.0 < capacity <= 1.0: + raise DataValidationError("Capacity must be in (0, 1]") + selected_count = capacity * len(targets) + ordered = sorted(zip(probabilities, targets), key=lambda item: item[0], reverse=True) + remaining = selected_count + selected_positive = 0.0 + index = 0 + while index < len(ordered) and remaining > 0.0: + score = ordered[index][0] + group_targets: List[int] = [] + while index < len(ordered) and ordered[index][0] == score: + group_targets.append(ordered[index][1]) + index += 1 + fraction = min(1.0, remaining / len(group_targets)) + selected_positive += fraction * sum(group_targets) + remaining -= fraction * len(group_targets) + positives = sum(targets) + return { + "precision": selected_positive / selected_count, + "capture": None if positives == 0 else selected_positive / positives, + } + + +def binary_metrics(targets: Sequence[int], probabilities: Sequence[float]) -> Dict[str, object]: + _validate_probabilities(targets, probabilities) + epsilon = 1e-15 + brier = sum( + (probability - target) ** 2 + for target, probability in zip(targets, probabilities) + ) / len(targets) + log_loss = -sum( + target * math.log(min(1.0 - epsilon, max(epsilon, probability))) + + (1 - target) + * math.log(min(1.0 - epsilon, max(epsilon, 1.0 - probability))) + for target, probability in zip(targets, probabilities) + ) / len(targets) + top_5 = top_capacity_metrics(targets, probabilities, 0.05) + top_10 = top_capacity_metrics(targets, probabilities, 0.10) + return { + "average_precision": average_precision(targets, probabilities), + "roc_auc": roc_auc(targets, probabilities), + "brier": brier, + "log_loss": log_loss, + "top_5_precision": top_5["precision"], + "top_5_capture": top_5["capture"], + "top_10_precision": top_10["precision"], + "top_10_capture": top_10["capture"], + } + + +def calibration_bins( + targets: Sequence[int], probabilities: Sequence[float], bin_count: int +) -> List[Dict[str, object]]: + _validate_probabilities(targets, probabilities) + if bin_count < 2: + raise DataValidationError("Calibration requires at least two bins") + bin_count = min(bin_count, len(targets)) + ordered = sorted(zip(probabilities, targets), key=lambda item: item[0]) + base_size, extra = divmod(len(ordered), bin_count) + result: List[Dict[str, object]] = [] + start = 0 + for bin_index in range(bin_count): + size = base_size + (1 if bin_index < extra else 0) + values = ordered[start : start + size] + start += size + result.append( + { + "bin": bin_index + 1, + "count": size, + "mean_probability": sum(item[0] for item in values) / size, + "observed_nonpass_rate": sum(item[1] for item in values) / size, + "min_probability": values[0][0], + "max_probability": values[-1][0], + } + ) + return result + + +def evaluation_cohorts( + rows: Sequence[EpisodeRow], include_audit_breakout: bool +) -> List[Tuple[str, List[EpisodeRow]]]: + if not include_audit_breakout: + return [("non_audit", _non_audit(rows))] + return [ + ("all", list(rows)), + ("non_audit", _non_audit(rows)), + ("never_fit_audit", [row for row in rows if row.audit_vehicle]), + ] + + +def evaluate_models( + mart: MartData, + trained: TrainedBaselines, + evaluate_locked: bool, + bin_count: int, +) -> Tuple[List[Dict[str, object]], List[Dict[str, object]]]: + metric_rows: List[Dict[str, object]] = [] + calibration_rows: List[Dict[str, object]] = [] + partitions = ["train", "tune", "calibrate"] + if evaluate_locked: + partitions.append("locked_test") + + for partition in partitions: + source_rows = mart.rows_by_partition[partition] + for cohort_name, rows in evaluation_cohorts( + source_rows, include_audit_breakout=(partition == "locked_test") + ): + if not rows: + continue + targets = _targets(rows) + model_probabilities: List[Tuple[str, List[float], int]] = [] + model_probabilities.append( + ( + "training_prevalence", + [trained.training_prevalence] * len(rows), + 0, + ) + ) + previous, fallback_count = literal_previous_probabilities( + rows, trained.training_prevalence + ) + model_probabilities.append( + ("previous_episode_literal", previous, fallback_count) + ) + model_probabilities.append( + ("logistic_raw", logistic_probabilities(trained, rows, calibrated=False), 0) + ) + if partition in {"calibrate", "locked_test"}: + model_probabilities.append( + ( + "logistic_platt", + logistic_probabilities(trained, rows, calibrated=True), + 0, + ) + ) + + for model_name, probabilities, fallback in model_probabilities: + metrics = binary_metrics(targets, probabilities) + metric_rows.append( + { + "model": model_name, + "partition": partition, + "cohort": cohort_name, + "episodes": len(rows), + "vehicles": len({row.vehicle_token for row in rows}), + "nonpass": sum(targets), + "previous_outcome_fallbacks": fallback, + **metrics, + } + ) + for bin_values in calibration_bins(targets, probabilities, bin_count): + calibration_rows.append( + { + "model": model_name, + "partition": partition, + "cohort": cohort_name, + **bin_values, + } + ) + return metric_rows, calibration_rows + + +def manifest_audit_counts( + mart: MartData, evaluate_locked: bool +) -> Tuple[ + Dict[str, Dict[str, object]], + Dict[str, Dict[str, Dict[str, object]]], + Dict[str, Dict[str, int]], +]: + """Return non-sensitive counts while withholding locked labels by default.""" + partition_counts: Dict[str, Dict[str, object]] = {} + source_era_counts: Dict[str, Dict[str, Dict[str, object]]] = {} + label_source_counts: Dict[str, Dict[str, int]] = {} + for partition, rows in mart.rows_by_partition.items(): + locked_and_closed = partition == "locked_test" and not evaluate_locked + partition_values: Dict[str, object] = { + "episodes": len(rows), + "audit_episodes": sum(row.audit_vehicle for row in rows), + } + if locked_and_closed: + partition_values["labels_withheld"] = True + else: + partition_values["nonpass"] = sum(row.target for row in rows) + partition_counts[partition] = partition_values + + partition_eras: Dict[str, Dict[str, object]] = {} + for row in rows: + era = row.source_era + if era is None: # guarded during mart validation + continue + values = partition_eras.setdefault(era, {"episodes": 0}) + values["episodes"] = int(values["episodes"]) + 1 + if not locked_and_closed: + values["nonpass"] = int(values.get("nonpass", 0)) + row.target + source_era_counts[partition] = partition_eras + label_source_counts[partition] = dict( + sorted(Counter(row.target_outcome_label_source for row in rows).items()) + ) + return partition_counts, source_era_counts, label_source_counts + + +def _is_private_project_path(path: Path) -> bool: + resolved = path.resolve() + private_roots = ( + (PROJECT_ROOT / "data/private").resolve(), + (PROJECT_ROOT / "artifacts/private").resolve(), + ) + for root in private_roots: + try: + resolved.relative_to(root) + return True + except ValueError: + continue + return False + + +def require_private_path(path: Path, label: str) -> Path: + resolved = path.resolve() + if not _is_private_project_path(resolved): + raise DataValidationError( + "{} must remain under data/private or artifacts/private".format(label) + ) + return resolved + + +def _atomic_text(path: Path, content: str) -> None: + partial = path.with_name(path.name + ".partial") + partial.write_text(content, encoding="utf-8") + os.replace(partial, path) + + +def _atomic_csv(path: Path, rows: Sequence[Mapping[str, object]]) -> None: + if not rows: + raise DataValidationError("Refusing to write an empty CSV artifact") + partial = path.with_name(path.name + ".partial") + columns = list(rows[0].keys()) + with partial.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="raise") + writer.writeheader() + writer.writerows(rows) + os.replace(partial, path) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + block = handle.read(1024 * 1024) + if not block: + break + digest.update(block) + return digest.hexdigest() + + +def write_artifacts( + output_dir: Path, + mart_path: Path, + mart: MartData, + trained: TrainedBaselines, + metric_rows: Sequence[Mapping[str, object]], + calibration_rows: Sequence[Mapping[str, object]], + evaluate_locked: bool, + seed: int, + overwrite: bool, + sklearn_version: str, +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + expected = ( + output_dir / "model.joblib", + output_dir / "metrics.json", + output_dir / "calibration_bins.csv", + output_dir / "manifest.json", + ) + existing = [path for path in expected if path.exists()] + if existing and not overwrite: + raise DataValidationError( + "Refusing to overwrite existing artifacts: " + + ", ".join(path.name for path in existing) + ) + + try: + import joblib + except ImportError as exc: + raise DependencyError("Writing the fitted model requires joblib") from exc + + model_path = output_dir / "model.joblib" + partial_model = model_path.with_name(model_path.name + ".partial") + joblib.dump( + { + "model_version": MODEL_VERSION, + "target_contract": TARGET_CONTRACT, + "platt_calibration_config": PLATT_CALIBRATION_CONFIG, + "numeric_features": NUMERIC_FEATURES, + "categorical_features": CATEGORICAL_FEATURES, + "training_prevalence": trained.training_prevalence, + "selected_c": trained.selected_c, + "convergence": { + "max_iter": trained.max_iter, + "selected_n_iter": trained.selected_n_iter, + "platt_n_iter": trained.platt_n_iter, + }, + "encoder": trained.encoder.artifact_state(), + "logistic_model": trained.logistic_model, + "platt_model": trained.platt_model, + }, + partial_model, + ) + os.replace(partial_model, model_path) + + _atomic_text( + output_dir / "metrics.json", + json.dumps(list(metric_rows), indent=2, sort_keys=True, allow_nan=False) + "\n", + ) + _atomic_csv(output_dir / "calibration_bins.csv", calibration_rows) + + partition_counts, source_era_counts, label_source_counts = manifest_audit_counts( + mart, evaluate_locked=evaluate_locked + ) + manifest = { + "model_version": MODEL_VERSION, + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "classification": "private_model_artifact_no_row_predictions", + "input_file": mart_path.name, + "input_sha256": _sha256(mart_path), + "metrics_sha256": _sha256(output_dir / "metrics.json"), + "input_rows": mart.input_rows, + "eligible_rows": mart.eligible_rows, + "ineligible_or_out_of_scope_rows": mart.ineligible_rows, + "partition_counts": partition_counts, + "source_era_audit_counts": source_era_counts, + "target_label_source_counts": label_source_counts, + "target_contract": TARGET_CONTRACT, + "platt_calibration_config": PLATT_CALIBRATION_CONFIG, + "split_bounds": { + name: {"start_inclusive": start.isoformat(), "end_exclusive": end.isoformat()} + for name, (start, end) in SPLIT_BOUNDS.items() + if name != "shadow" + }, + "never_fit_audit_rule": "mart is_vin_audit; validated as vehicle_bucket < 10", + "locked_test_evaluated": evaluate_locked, + "numeric_features": NUMERIC_FEATURES, + "categorical_features": CATEGORICAL_FEATURES, + "selected_c": trained.selected_c, + "convergence": { + "max_iter": trained.max_iter, + "selected_n_iter": trained.selected_n_iter, + "platt_n_iter": trained.platt_n_iter, + "artifact_acceptance_requires_convergence": True, + "artifact_acceptance_requires_finite_coefficients_and_probabilities": True, + }, + "tuning_results": trained.tuning_results, + "seed": seed, + "python_version": sys.version, + "scikit_learn_version": sklearn_version, + "artifacts": [path.name for path in expected], + } + _atomic_text( + output_dir / "manifest.json", + json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parse_args(argv) + os.umask(0o077) + try: + mart_path = require_private_path(args.mart, "--mart") + output_dir = require_private_path(args.output_dir, "--output-dir") + c_grid = parse_c_grid(args.c_grid) + if args.max_iter < 1: + raise DataValidationError("--max-iter must be positive") + if args.calibration_bins < 2: + raise DataValidationError("--calibration-bins must be at least 2") + mart = load_mart(mart_path) + trained = train_baselines( + mart, + c_grid=c_grid, + min_category_count=args.min_category_count, + seed=args.seed, + max_iter=args.max_iter, + ) + metric_rows, calibration_rows = evaluate_models( + mart, + trained, + evaluate_locked=args.evaluate_locked, + bin_count=args.calibration_bins, + ) + ( + _np, + sklearn, + _dict_vectorizer, + _logistic, + _convergence_warning, + ) = require_ml_dependencies() + write_artifacts( + output_dir=output_dir, + mart_path=mart_path, + mart=mart, + trained=trained, + metric_rows=metric_rows, + calibration_rows=calibration_rows, + evaluate_locked=args.evaluate_locked, + seed=args.seed, + overwrite=args.overwrite, + sklearn_version=sklearn.__version__, + ) + except BaselineError as exc: + print("Baseline training failed: {}".format(exc), file=sys.stderr) + return 2 + + print("Wrote private baseline artifacts to {}".format(output_dir)) + if args.evaluate_locked: + print("The explicitly unlocked 2025 test metrics were evaluated.") + else: + print("The 2025 locked test was not evaluated.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/train_tree_model.py b/scripts/train_tree_model.py new file mode 100644 index 0000000..2c4aad3 --- /dev/null +++ b/scripts/train_tree_model.py @@ -0,0 +1,799 @@ +"""Train a leakage-safe nonlinear candidate from the private episode mart. + +The model is fitted only on non-audit 2016-2022 episodes, selected on 2023, +and probability-calibrated on 2024. The locked 2025 partition is evaluated +only when ``--evaluate-locked`` is supplied. No row-level predictions are +written. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import statistics +import sys +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Mapping, Optional, Sequence, Tuple + +try: + from scripts import train_baselines as baselines +except ModuleNotFoundError: # Direct execution places scripts/ on sys.path. + import train_baselines as baselines + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MODEL_VERSION = "hist_gradient_boosting_v1" +DEFAULT_OUTPUT = PROJECT_ROOT / "artifacts/private/tree" / MODEL_VERSION + +NUMERIC_FEATURES = baselines.NUMERIC_FEATURES +CATEGORICAL_FEATURES = baselines.CATEGORICAL_FEATURES + +MISSING_CATEGORY_CODE = 0 +RARE_CATEGORY_CODE = 1 +UNKNOWN_CATEGORY_CODE = 2 +FIRST_KNOWN_CATEGORY_CODE = 3 +MAX_HISTOGRAM_BINS = 255 + +# These columns are validated by the baseline mart loader but are deliberately +# unavailable to the nonlinear feature encoder. +EXCLUDED_FROM_PREDICTORS = ( + "vehicle_token", + "vehicle_bucket", + "is_vin_audit", + "episode_number", + "episode_start", + "first_outcome", + "target_nonpass", + "eligible_returning_target", + "temporal_partition", + "source_era", + "target_outcome_label_source", +) + + +@dataclass(frozen=True) +class TreeCandidate: + learning_rate: float + max_leaf_nodes: int + l2_regularization: float + + def artifact_state(self) -> Dict[str, object]: + return { + "learning_rate": self.learning_rate, + "max_leaf_nodes": self.max_leaf_nodes, + "l2_regularization": self.l2_regularization, + } + + +@dataclass +class TreeFeatureEncoder: + """Train-only median imputation and bounded ordinal category encoding.""" + + min_category_count: int + max_categories: int + numeric_medians: Dict[str, float] + seen_categories: Dict[str, set] + known_category_codes: Dict[str, Dict[str, int]] + feature_names: Tuple[str, ...] + categorical_mask: Tuple[bool, ...] + + @classmethod + def fit( + cls, + rows: Sequence[baselines.EpisodeRow], + min_category_count: int, + max_categories: int, + ) -> "TreeFeatureEncoder": + if not rows: + raise baselines.DataValidationError( + "Cannot fit tree preprocessing on zero rows" + ) + if min_category_count < 1: + raise baselines.DataValidationError( + "min_category_count must be at least 1" + ) + if not FIRST_KNOWN_CATEGORY_CODE + 1 <= max_categories <= MAX_HISTOGRAM_BINS: + raise baselines.DataValidationError( + "max_categories must be between {} and {}".format( + FIRST_KNOWN_CATEGORY_CODE + 1, MAX_HISTOGRAM_BINS + ) + ) + + numeric_medians: Dict[str, float] = {} + for name in NUMERIC_FEATURES: + observed = [ + row.numeric[name] + for row in rows + if row.numeric[name] is not None + ] + if not observed: + raise baselines.DataValidationError( + "Training data has no observed values for numeric feature " + name + ) + median = float(statistics.median(observed)) + if not math.isfinite(median): + raise baselines.DataValidationError( + "Training median is non-finite for numeric feature " + name + ) + numeric_medians[name] = median + + seen_categories: Dict[str, set] = {} + known_category_codes: Dict[str, Dict[str, int]] = {} + category_capacity = max_categories - FIRST_KNOWN_CATEGORY_CODE + for name in CATEGORICAL_FEATURES: + observed_values = [ + row.categorical[name] + for row in rows + if row.categorical[name] is not None + ] + counts = Counter(observed_values) + seen_categories[name] = set(counts) + candidates = [ + (value, count) + for value, count in counts.items() + if count >= min_category_count + ] + # Frequency first, lexical second makes capping deterministic. + candidates.sort(key=lambda item: (-item[1], item[0])) + kept = candidates[:category_capacity] + known_category_codes[name] = { + value: FIRST_KNOWN_CATEGORY_CODE + index + for index, (value, _count) in enumerate(kept) + } + + feature_names = tuple( + list(NUMERIC_FEATURES) + + ["missing__" + name for name in NUMERIC_FEATURES] + + list(CATEGORICAL_FEATURES) + ) + categorical_mask = tuple( + [False] * (2 * len(NUMERIC_FEATURES)) + + [True] * len(CATEGORICAL_FEATURES) + ) + return cls( + min_category_count=min_category_count, + max_categories=max_categories, + numeric_medians=numeric_medians, + seen_categories=seen_categories, + known_category_codes=known_category_codes, + feature_names=feature_names, + categorical_mask=categorical_mask, + ) + + def category_code(self, feature: str, value: Optional[str]) -> int: + if value is None: + return MISSING_CATEGORY_CODE + known = self.known_category_codes[feature] + if value in known: + return known[value] + if value in self.seen_categories[feature]: + return RARE_CATEGORY_CODE + return UNKNOWN_CATEGORY_CODE + + def transform(self, rows: Sequence[baselines.EpisodeRow], np_module: object) -> object: + matrix = np_module.empty((len(rows), len(self.feature_names)), dtype=np_module.float64) + for row_index, row in enumerate(rows): + column = 0 + for name in NUMERIC_FEATURES: + value = row.numeric[name] + matrix[row_index, column] = ( + self.numeric_medians[name] if value is None else float(value) + ) + column += 1 + for name in NUMERIC_FEATURES: + matrix[row_index, column] = 1.0 if row.numeric[name] is None else 0.0 + column += 1 + for name in CATEGORICAL_FEATURES: + matrix[row_index, column] = float( + self.category_code(name, row.categorical[name]) + ) + column += 1 + if matrix.size and not bool(np_module.isfinite(matrix).all()): + raise baselines.DataValidationError( + "Tree feature matrix contains non-finite values" + ) + return matrix + + def artifact_state(self) -> Dict[str, object]: + return { + "min_category_count": self.min_category_count, + "max_categories": self.max_categories, + "reserved_category_codes": { + "missing": MISSING_CATEGORY_CODE, + "rare_seen_in_training": RARE_CATEGORY_CODE, + "unknown_after_training": UNKNOWN_CATEGORY_CODE, + "first_known": FIRST_KNOWN_CATEGORY_CODE, + }, + "numeric_medians": self.numeric_medians, + "seen_categories": { + name: sorted(values) for name, values in self.seen_categories.items() + }, + "known_category_codes": self.known_category_codes, + "feature_names": self.feature_names, + "categorical_mask": self.categorical_mask, + } + + +@dataclass +class TrainedTreeModel: + encoder: TreeFeatureEncoder + model: object + platt_model: object + selected_candidate: TreeCandidate + model_n_iter: int + platt_n_iter: int + max_iter: int + calibration_max_iter: int + tuning_results: List[Dict[str, object]] + + +def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mart", required=True, type=Path) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--seed", type=int, default=20260715) + parser.add_argument("--min-category-count", type=int, default=100) + parser.add_argument("--max-categories", type=int, default=128) + parser.add_argument("--learning-rates", default="0.05,0.1") + parser.add_argument("--max-leaf-nodes", default="15,31") + parser.add_argument("--l2-grid", default="1.0") + parser.add_argument("--min-samples-leaf", type=int, default=20) + parser.add_argument("--max-iter", type=int, default=300) + parser.add_argument("--n-iter-no-change", type=int, default=20) + parser.add_argument("--calibration-max-iter", type=int, default=5000) + parser.add_argument("--calibration-bins", type=int, default=10) + parser.add_argument("--evaluate-locked", action="store_true") + parser.add_argument("--overwrite", action="store_true") + return parser.parse_args(argv) + + +def _parse_float_grid(value: str, label: str, allow_zero: bool) -> List[float]: + try: + values = [float(part.strip()) for part in value.split(",") if part.strip()] + except ValueError as exc: + raise baselines.DataValidationError(label + " must contain finite numbers") from exc + minimum_ok = (lambda item: item >= 0.0) if allow_zero else (lambda item: item > 0.0) + if not values or any(not math.isfinite(item) or not minimum_ok(item) for item in values): + raise baselines.DataValidationError(label + " contains an invalid value") + return sorted(set(values)) + + +def _parse_int_grid(value: str, label: str, minimum: int) -> List[int]: + try: + values = [int(part.strip()) for part in value.split(",") if part.strip()] + except ValueError as exc: + raise baselines.DataValidationError(label + " must contain integers") from exc + if not values or any(item < minimum for item in values): + raise baselines.DataValidationError(label + " contains an invalid value") + return sorted(set(values)) + + +def candidate_grid( + learning_rates: Sequence[float], + max_leaf_nodes: Sequence[int], + l2_values: Sequence[float], +) -> List[TreeCandidate]: + return [ + TreeCandidate(rate, leaves, l2) + for rate in sorted(set(learning_rates)) + for leaves in sorted(set(max_leaf_nodes)) + for l2 in sorted(set(l2_values)) + ] + + +def require_tree_dependencies() -> Tuple[object, object, object, object, object, object]: + # joblib's macOS physical-core probe emits a UserWarning on this managed + # host. Preserve an explicit user setting; otherwise give it the already + # available logical count so warning-as-error validation remains usable. + os.environ.setdefault("LOKY_MAX_CPU_COUNT", "1") + try: + import joblib + import numpy as np + import sklearn + from sklearn.ensemble import HistGradientBoostingClassifier + from sklearn.exceptions import ConvergenceWarning + from sklearn.linear_model import LogisticRegression + except ImportError as exc: + raise baselines.DependencyError( + "Tree training requires numpy, scikit-learn, and joblib" + ) from exc + return ( + np, + sklearn, + joblib, + HistGradientBoostingClassifier, + LogisticRegression, + ConvergenceWarning, + ) + + +def _non_audit(rows: Sequence[baselines.EpisodeRow]) -> List[baselines.EpisodeRow]: + return [row for row in rows if not row.audit_vehicle] + + +def _targets(rows: Sequence[baselines.EpisodeRow]) -> List[int]: + return [row.target for row in rows] + + +def _scores_finite(model: object, np_module: object) -> bool: + for attribute in ("train_score_", "validation_score_"): + values = getattr(model, attribute, None) + if values is None: + continue + array = np_module.asarray(values) + if array.size and not bool(np_module.isfinite(array).all()): + return False + return True + + +def _hist_converged( + model: object, + convergence_messages: Sequence[str], + max_iter: int, + np_module: object, +) -> Tuple[bool, Optional[int]]: + value = getattr(model, "n_iter_", None) + if value is None: + return False, None + try: + n_iter = int(value) + except (TypeError, ValueError, OverflowError): + return False, None + converged = ( + not convergence_messages + and 0 < n_iter < max_iter + and _scores_finite(model, np_module) + ) + return converged, n_iter + + +def _decision_scores(model: object, features: object, np_module: object) -> object: + scores = np_module.asarray(model.decision_function(features), dtype=np_module.float64) + scores = scores.reshape(-1, 1) + if scores.size == 0 or not bool(np_module.isfinite(scores).all()): + raise baselines.DataValidationError( + "Tree model produced non-finite decision scores" + ) + return scores + + +def train_tree_model( + mart: baselines.MartData, + candidates: Sequence[TreeCandidate], + min_category_count: int, + max_categories: int, + min_samples_leaf: int, + max_iter: int, + n_iter_no_change: int, + calibration_max_iter: int, + seed: int, +) -> TrainedTreeModel: + ( + np, + _sklearn, + _joblib, + hist_class, + logistic_class, + convergence_warning_class, + ) = require_tree_dependencies() + if not candidates: + raise baselines.DataValidationError("The tree tuning grid is empty") + if min_samples_leaf < 1 or max_iter < 2 or n_iter_no_change < 1: + raise baselines.DataValidationError("Invalid tree iteration/leaf configuration") + if calibration_max_iter < 2: + raise baselines.DataValidationError("calibration_max_iter must be at least 2") + + train_rows = _non_audit(mart.rows_by_partition["train"]) + tune_rows = _non_audit(mart.rows_by_partition["tune"]) + calibrate_rows = _non_audit(mart.rows_by_partition["calibrate"]) + baselines._require_two_classes(train_rows, "Tree training") + baselines._require_two_classes(tune_rows, "Tree tuning") + baselines._require_two_classes(calibrate_rows, "Tree calibration") + + encoder = TreeFeatureEncoder.fit( + train_rows, + min_category_count=min_category_count, + max_categories=max_categories, + ) + x_train = encoder.transform(train_rows, np) + x_tune = encoder.transform(tune_rows, np) + x_calibrate = encoder.transform(calibrate_rows, np) + train_targets = np.asarray(_targets(train_rows), dtype=np.int8) + tune_targets = _targets(tune_rows) + calibrate_targets = np.asarray(_targets(calibrate_rows), dtype=np.int8) + + selectable: List[Tuple[float, float, float, int, float, object, int, TreeCandidate]] = [] + tuning_results: List[Dict[str, object]] = [] + for candidate in candidates: + model = hist_class( + loss="log_loss", + learning_rate=candidate.learning_rate, + max_iter=max_iter, + max_leaf_nodes=candidate.max_leaf_nodes, + min_samples_leaf=min_samples_leaf, + l2_regularization=candidate.l2_regularization, + max_bins=MAX_HISTOGRAM_BINS, + categorical_features=list(encoder.categorical_mask), + early_stopping=True, + scoring="loss", + validation_fraction=0.10, + n_iter_no_change=n_iter_no_change, + tol=1e-7, + random_state=seed, + class_weight=None, + ) + result: Dict[str, object] = { + **candidate.artifact_state(), + "converged": False, + "n_iter": None, + "convergence_warning": None, + "finite_scores": False, + "finite_train_probabilities": False, + "finite_tune_probabilities": False, + "eligible_for_selection": False, + "brier": None, + "average_precision": None, + } + convergence_messages = baselines.fit_with_convergence_capture( + model, x_train, train_targets, convergence_warning_class + ) + converged, n_iter = _hist_converged( + model, convergence_messages, max_iter, np + ) + result["converged"] = converged + result["n_iter"] = n_iter + result["convergence_warning"] = ( + " | ".join(convergence_messages) if convergence_messages else None + ) + result["finite_scores"] = _scores_finite(model, np) + + train_probabilities = model.predict_proba(x_train)[:, 1] + tune_probabilities = model.predict_proba(x_tune)[:, 1] + finite_train = baselines.probability_array_finite(train_probabilities, np) + finite_tune = baselines.probability_array_finite(tune_probabilities, np) + result["finite_train_probabilities"] = finite_train + result["finite_tune_probabilities"] = finite_tune + eligible = converged and finite_train and finite_tune + result["eligible_for_selection"] = eligible + if eligible and n_iter is not None: + metrics = baselines.binary_metrics(tune_targets, tune_probabilities.tolist()) + brier = float(metrics["brier"]) + average_precision = float(metrics["average_precision"]) + result["brier"] = brier + result["average_precision"] = average_precision + selectable.append( + ( + brier, + -average_precision, + candidate.learning_rate, + candidate.max_leaf_nodes, + candidate.l2_regularization, + model, + n_iter, + candidate, + ) + ) + tuning_results.append(result) + + if not selectable: + raise baselines.DataValidationError( + "No tree candidate converged with finite train/tune probabilities; " + "increase --max-iter or inspect the feature contract" + ) + ( + _brier, + _negative_ap, + _rate, + _leaves, + _l2, + selected_model, + selected_n_iter, + selected_candidate, + ) = min(selectable, key=lambda item: item[:5]) + + calibration_scores = _decision_scores(selected_model, x_calibrate, np) + platt_model = logistic_class( + C=baselines.PLATT_CALIBRATION_CONFIG["c"], + penalty=baselines.PLATT_CALIBRATION_CONFIG["penalty"], + solver=baselines.PLATT_CALIBRATION_CONFIG["solver"], + class_weight=None, + max_iter=calibration_max_iter, + random_state=seed, + ) + platt_warnings = baselines.fit_with_convergence_capture( + platt_model, + calibration_scores, + calibrate_targets, + convergence_warning_class, + ) + platt_n_iter = baselines.model_n_iter(platt_model, np) + if ( + platt_warnings + or platt_n_iter is None + or platt_n_iter >= calibration_max_iter + or not baselines.model_parameters_finite(platt_model, np) + ): + detail = " | ".join(platt_warnings) if platt_warnings else "finite/convergence check" + raise baselines.DataValidationError( + "Tree Platt calibration did not converge: " + detail + ) + calibrated = platt_model.predict_proba(calibration_scores)[:, 1] + if not baselines.probability_array_finite(calibrated, np): + raise baselines.DataValidationError( + "Tree Platt calibration produced non-finite probabilities" + ) + + return TrainedTreeModel( + encoder=encoder, + model=selected_model, + platt_model=platt_model, + selected_candidate=selected_candidate, + model_n_iter=selected_n_iter, + platt_n_iter=int(platt_n_iter), + max_iter=max_iter, + calibration_max_iter=calibration_max_iter, + tuning_results=tuning_results, + ) + + +def tree_probabilities( + trained: TrainedTreeModel, + rows: Sequence[baselines.EpisodeRow], + calibrated: bool, +) -> List[float]: + np, _sklearn, _joblib, _hist, _logistic, _warning = require_tree_dependencies() + matrix = trained.encoder.transform(rows, np) + if calibrated: + scores = _decision_scores(trained.model, matrix, np) + probabilities = trained.platt_model.predict_proba(scores)[:, 1] + else: + probabilities = trained.model.predict_proba(matrix)[:, 1] + if not baselines.probability_array_finite(probabilities, np): + raise baselines.DataValidationError( + "Tree model produced non-finite probabilities" + ) + return probabilities.tolist() + + +def evaluate_tree_model( + mart: baselines.MartData, + trained: TrainedTreeModel, + evaluate_locked: bool, + bin_count: int, +) -> Tuple[List[Dict[str, object]], List[Dict[str, object]]]: + metric_rows: List[Dict[str, object]] = [] + calibration_rows: List[Dict[str, object]] = [] + partitions = ["train", "tune", "calibrate"] + if evaluate_locked: + partitions.append("locked_test") + + for partition in partitions: + source_rows = mart.rows_by_partition[partition] + cohorts = baselines.evaluation_cohorts( + source_rows, include_audit_breakout=(partition == "locked_test") + ) + for cohort_name, rows in cohorts: + if not rows: + continue + targets = _targets(rows) + predictions = [ + ( + "hist_gradient_boosting_raw", + tree_probabilities(trained, rows, calibrated=False), + ) + ] + if partition in {"calibrate", "locked_test"}: + predictions.append( + ( + "hist_gradient_boosting_platt", + tree_probabilities(trained, rows, calibrated=True), + ) + ) + for model_name, probabilities in predictions: + metrics = baselines.binary_metrics(targets, probabilities) + metric_rows.append( + { + "model": model_name, + "partition": partition, + "cohort": cohort_name, + "episodes": len(rows), + "vehicles": len({row.vehicle_token for row in rows}), + "nonpass": sum(targets), + **metrics, + } + ) + for values in baselines.calibration_bins( + targets, probabilities, bin_count + ): + calibration_rows.append( + { + "model": model_name, + "partition": partition, + "cohort": cohort_name, + **values, + } + ) + return metric_rows, calibration_rows + + +def _input_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def write_artifacts( + output_dir: Path, + mart_path: Path, + mart: baselines.MartData, + trained: TrainedTreeModel, + metric_rows: Sequence[Mapping[str, object]], + calibration_rows: Sequence[Mapping[str, object]], + evaluate_locked: bool, + seed: int, + overwrite: bool, + sklearn_version: str, +) -> None: + _np, _sklearn, joblib, _hist, _logistic, _warning = require_tree_dependencies() + output_dir.mkdir(parents=True, exist_ok=True) + model_path = output_dir / "model.joblib" + metrics_path = output_dir / "metrics.json" + calibration_path = output_dir / "calibration_bins.csv" + manifest_path = output_dir / "manifest.json" + expected = (model_path, metrics_path, calibration_path, manifest_path) + existing = [path for path in expected if path.exists()] + if existing and not overwrite: + raise baselines.DataValidationError( + "Refusing to overwrite existing tree artifacts: " + + ", ".join(path.name for path in existing) + ) + + partial_model = model_path.with_name(model_path.name + ".partial") + joblib.dump( + { + "model_version": MODEL_VERSION, + "target_contract": baselines.TARGET_CONTRACT, + "numeric_features": NUMERIC_FEATURES, + "categorical_features": CATEGORICAL_FEATURES, + "excluded_from_predictors": EXCLUDED_FROM_PREDICTORS, + "encoder": trained.encoder.artifact_state(), + "selected_candidate": trained.selected_candidate.artifact_state(), + "hist_gradient_boosting_model": trained.model, + "platt_model": trained.platt_model, + "platt_calibration_config": baselines.PLATT_CALIBRATION_CONFIG, + }, + partial_model, + ) + os.replace(partial_model, model_path) + baselines._atomic_text( + metrics_path, + json.dumps(list(metric_rows), indent=2, sort_keys=True, allow_nan=False) + + "\n", + ) + baselines._atomic_csv(calibration_path, calibration_rows) + + partition_counts, source_counts, label_source_counts = ( + baselines.manifest_audit_counts(mart, evaluate_locked=evaluate_locked) + ) + manifest = { + "model_version": MODEL_VERSION, + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "classification": "private_model_artifact_no_row_predictions", + "input_file": mart_path.name, + "input_sha256": _input_sha256(mart_path), + "metrics_sha256": _input_sha256(metrics_path), + "input_rows": mart.input_rows, + "eligible_rows": mart.eligible_rows, + "partition_counts": partition_counts, + "source_era_audit_counts": source_counts, + "target_label_source_counts": label_source_counts, + "target_contract": baselines.TARGET_CONTRACT, + "never_fit_audit_rule": "mart is_vin_audit; validated as vehicle_bucket < 10", + "locked_test_evaluated": evaluate_locked, + "split_bounds": { + name: { + "start_inclusive": start.isoformat(), + "end_exclusive": end.isoformat(), + } + for name, (start, end) in baselines.SPLIT_BOUNDS.items() + if name != "shadow" + }, + "numeric_features": NUMERIC_FEATURES, + "categorical_features": CATEGORICAL_FEATURES, + "excluded_from_predictors": EXCLUDED_FROM_PREDICTORS, + "preprocessing_fit_partition": "train_2016_2022_non_audit_only", + "unknown_category_code": UNKNOWN_CATEGORY_CODE, + "selected_candidate": trained.selected_candidate.artifact_state(), + "convergence": { + "tree_max_iter": trained.max_iter, + "tree_selected_n_iter": trained.model_n_iter, + "platt_max_iter": trained.calibration_max_iter, + "platt_n_iter": trained.platt_n_iter, + "requires_early_stop_before_max_iter": True, + "requires_finite_scores_and_probabilities": True, + }, + "platt_calibration_config": baselines.PLATT_CALIBRATION_CONFIG, + "tuning_results": trained.tuning_results, + "seed": seed, + "python_version": sys.version, + "scikit_learn_version": sklearn_version, + "artifacts": [path.name for path in expected], + } + baselines._atomic_text( + manifest_path, + json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parse_args(argv) + os.umask(0o077) + try: + mart_path = baselines.require_private_path(args.mart, "--mart") + output_dir = baselines.require_private_path(args.output_dir, "--output-dir") + learning_rates = _parse_float_grid( + args.learning_rates, "--learning-rates", allow_zero=False + ) + leaf_nodes = _parse_int_grid( + args.max_leaf_nodes, "--max-leaf-nodes", minimum=2 + ) + l2_values = _parse_float_grid(args.l2_grid, "--l2-grid", allow_zero=True) + if args.calibration_bins < 2: + raise baselines.DataValidationError( + "--calibration-bins must be at least 2" + ) + mart = baselines.load_mart(mart_path) + trained = train_tree_model( + mart=mart, + candidates=candidate_grid(learning_rates, leaf_nodes, l2_values), + min_category_count=args.min_category_count, + max_categories=args.max_categories, + min_samples_leaf=args.min_samples_leaf, + max_iter=args.max_iter, + n_iter_no_change=args.n_iter_no_change, + calibration_max_iter=args.calibration_max_iter, + seed=args.seed, + ) + metric_rows, calibration_rows = evaluate_tree_model( + mart, + trained, + evaluate_locked=args.evaluate_locked, + bin_count=args.calibration_bins, + ) + _np, sklearn, _joblib, _hist, _logistic, _warning = ( + require_tree_dependencies() + ) + write_artifacts( + output_dir=output_dir, + mart_path=mart_path, + mart=mart, + trained=trained, + metric_rows=metric_rows, + calibration_rows=calibration_rows, + evaluate_locked=args.evaluate_locked, + seed=args.seed, + overwrite=args.overwrite, + sklearn_version=sklearn.__version__, + ) + except baselines.BaselineError as exc: + print("Tree training failed: {}".format(exc), file=sys.stderr) + return 2 + + print("Wrote private nonlinear model artifacts to {}".format(output_dir)) + if args.evaluate_locked: + print("The explicitly unlocked 2025 test metrics were evaluated.") + else: + print("The 2025 locked test was not evaluated.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sql/00_read_only_connection_check.sql b/sql/00_read_only_connection_check.sql new file mode 100644 index 0000000..eede70a --- /dev/null +++ b/sql/00_read_only_connection_check.sql @@ -0,0 +1,39 @@ +-- Run the whole file when connected to the countydata database. +-- The explicit read-only transaction protects against accidental writes. +BEGIN TRANSACTION READ ONLY; + +SELECT + current_database() AS database_name, + current_user AS database_user, + current_setting('transaction_read_only') AS transaction_read_only, + version() AS postgres_version; + +SELECT + ssl, + version AS tls_version, + cipher, + bits +FROM pg_stat_ssl +WHERE pid = pg_backend_pid(); + +SELECT + n.nspname AS schema_name, + c.relname AS relation_name, + CASE c.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'partitioned table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized view' + WHEN 'f' THEN 'foreign table' + ELSE c.relkind::text + END AS relation_type, + COALESCE(s.n_live_tup, c.reltuples)::bigint AS estimated_rows +FROM pg_class AS c +JOIN pg_namespace AS n ON n.oid = c.relnamespace +LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid +WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' +ORDER BY n.nspname, c.relname; + +ROLLBACK; diff --git a/sql/01_safe_data_overview.sql b/sql/01_safe_data_overview.sql new file mode 100644 index 0000000..a8065e7 --- /dev/null +++ b/sql/01_safe_data_overview.sql @@ -0,0 +1,53 @@ +-- Aggregate-only starter queries. Do not select VIN, plate, email, session, +-- upload-log, or raw JSON fields into a public-facing application. +BEGIN TRANSACTION READ ONLY; + +SELECT + count(*) AS registration_records, + min(registration_date) AS earliest_registration, + max(registration_date) AS latest_registration +FROM data.dmv_tax; + +SELECT + extract(year FROM registration_date)::integer AS registration_year, + count(*) AS records +FROM data.dmv_tax +GROUP BY 1 +ORDER BY 1; + +SELECT + upper(trim(county)) AS county, + count(*) AS records +FROM data.dmv_tax +GROUP BY 1 +ORDER BY 2 DESC; + +SELECT + upper(replace(trim(fuel_type), '-', ' ')) AS canonical_fuel_type, + count(*) AS records +FROM data.dmv_tax_vehicle +GROUP BY 1 +ORDER BY 2 DESC; + +SELECT + count(*) AS inspection_records, + min(test_start) AS earliest_test, + max(test_start) AS latest_test +FROM data.inspection_search; + +SELECT + extract(year FROM test_start)::integer AS test_year, + count(*) AS records +FROM data.inspection_search +GROUP BY 1 +ORDER BY 1; + +SELECT + lower(trim(county)) AS county_source, + coalesce(nullif(upper(trim(overall_result)), ''), '') AS result, + count(*) AS records +FROM data.inspection_search +GROUP BY 1, 2 +ORDER BY 1, 3 DESC; + +ROLLBACK; diff --git a/sql/10_episode_cohort_feasibility.sql b/sql/10_episode_cohort_feasibility.sql new file mode 100644 index 0000000..4c829f6 --- /dev/null +++ b/sql/10_episode_cohort_feasibility.sql @@ -0,0 +1,236 @@ +-- Aggregate-only feasibility profile for the selected project (label contract v3). +-- +-- This intentionally samples candidate VINs, keeps their complete histories, +-- and returns only aggregates. The sample is suitable for cohort development, +-- not for final population estimates. Run the whole file on countydata. +BEGIN TRANSACTION READ ONLY; + +WITH candidate_vins AS MATERIALIZED ( + SELECT DISTINCT vin + FROM data.inspection_search TABLESAMPLE SYSTEM (0.25) REPEATABLE (20260715) + WHERE vin IS NOT NULL + AND length(btrim(vin)) = 17 +), +sampled_vins AS MATERIALIZED ( + SELECT vin + FROM candidate_vins + ORDER BY md5(vin) + LIMIT 2000 +), +base AS MATERIALIZED ( + SELECT + s.vin, + s.id, + s.test_start, + lower(btrim(s.county)) AS source_era, + CASE + WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake' + ELSE lower(btrim(s.county)) + END AS public_county, + CASE upper(btrim(s.overall_result)) + WHEN 'PASS' THEN 'pass' + WHEN 'P' THEN 'pass' + WHEN 'FAIL' THEN 'fail' + WHEN 'F' THEN 'fail' + WHEN 'REJECT' THEN 'reject' + WHEN 'ABORT' THEN 'abort' + ELSE CASE + WHEN lower(btrim(s.county)) = 'utah' + AND lower(btrim(s.program_type)) = 'obd' + AND upper(btrim(s.test_type)) = 'OBD' THEN + CASE upper(btrim(s.obd_result)) + WHEN 'PASS' THEN 'pass' + WHEN 'P' THEN 'pass' + WHEN 'FAIL' THEN 'fail' + WHEN 'F' THEN 'fail' + WHEN 'REJECT' THEN 'reject' + WHEN 'ABORT' THEN 'abort' + ELSE NULL + END + ELSE NULL + END + END AS outcome, + CASE + WHEN upper(btrim(s.overall_result)) IN ( + 'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT' + ) THEN 'overall_result' + WHEN lower(btrim(s.county)) = 'utah' + AND lower(btrim(s.program_type)) = 'obd' + AND upper(btrim(s.test_type)) = 'OBD' + AND upper(btrim(s.obd_result)) IN ( + 'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT' + ) THEN 'utah_obd_proxy' + ELSE NULL + END AS outcome_label_source + -- Only Utah OBD/OBD rows may use utah_obd_proxy, and only for binary + -- pass/non-pass analysis. Multiclass analysis must require + -- outcome_label_source='overall_result'. + FROM data.inspection_search AS s + JOIN sampled_vins USING (vin) + WHERE s.test_start >= timestamp '2010-01-01' +), +timestamp_quality AS ( + SELECT + vin, + test_start, + count( + DISTINCT row( + source_era, + public_county, + coalesce(outcome, ''), + coalesce(outcome_label_source, '') + ) + ) AS analytical_variants_at_timestamp + FROM base + GROUP BY 1, 2 +), +timestamp_groups AS ( + SELECT + base.*, + row_number() OVER ( + PARTITION BY vin, test_start + ORDER BY id + ) AS duplicate_rank + FROM base +), +deduplicated AS ( + SELECT + g.vin, + g.id, + g.test_start, + g.source_era, + g.public_county, + g.outcome, + g.outcome_label_source + FROM timestamp_groups AS g + JOIN timestamp_quality AS q USING (vin, test_start) + WHERE g.duplicate_rank = 1 + AND q.analytical_variants_at_timestamp = 1 +), +with_gaps AS ( + SELECT + *, + lag(test_start) OVER ( + PARTITION BY vin ORDER BY test_start, id + ) AS previous_test_start + FROM deduplicated +), +episode_markers AS ( + SELECT + *, + CASE + WHEN previous_test_start IS NULL + OR test_start - previous_test_start > interval '30 days' + THEN 1 ELSE 0 + END AS starts_episode + FROM with_gaps +), +numbered AS ( + SELECT + *, + sum(starts_episode) OVER ( + PARTITION BY vin ORDER BY test_start, id + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS episode_number + FROM episode_markers +), +attempts AS ( + SELECT + *, + row_number() OVER ( + PARTITION BY vin, episode_number ORDER BY test_start, id + ) AS attempt_number + FROM numbered +), +episode_summaries AS ( + SELECT + vin, + episode_number, + min(test_start) AS episode_start, + count(*) AS attempt_count, + bool_or(outcome = 'pass') AS eventually_passed + FROM attempts + GROUP BY 1, 2 +), +episodes AS ( + SELECT + a.vin, + a.episode_number, + a.test_start AS episode_start, + a.source_era, + a.public_county, + a.outcome AS first_outcome, + a.outcome_label_source AS first_outcome_label_source, + s.attempt_count, + s.eventually_passed + FROM attempts AS a + JOIN episode_summaries AS s USING (vin, episode_number) + WHERE a.attempt_number = 1 +), +sequenced AS ( + SELECT + vin, + episode_number, + episode_start, + source_era, + public_county, + first_outcome, + first_outcome_label_source, + lag(episode_start) OVER ( + PARTITION BY vin ORDER BY episode_number + ) AS prior_episode_start, + lag(first_outcome) OVER ( + PARTITION BY vin ORDER BY episode_number + ) AS prior_first_outcome, + lag(attempt_count) OVER ( + PARTITION BY vin ORDER BY episode_number + ) AS prior_attempt_count, + sum(attempt_count) OVER ( + PARTITION BY vin ORDER BY episode_number + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING + ) AS prior_event_count + FROM episodes +), +eligible AS ( + SELECT * + FROM sequenced + WHERE episode_start >= timestamp '2016-01-01' + AND episode_number > 1 + AND first_outcome IS NOT NULL + AND prior_event_count <= 50 +) +SELECT + CASE + WHEN grouping(extract(year FROM episode_start)::integer) = 1 + THEN NULL + ELSE extract(year FROM episode_start)::integer + END AS target_year, + count(*) AS eligible_episodes, + count(DISTINCT vin) AS vehicles, + count(*) FILTER (WHERE first_outcome = 'pass') AS pass_episodes, + count(*) FILTER ( + WHERE first_outcome IN ('fail', 'reject', 'abort') + ) AS nonpass_episodes, + count(*) FILTER ( + WHERE first_outcome_label_source = 'utah_obd_proxy' + ) AS utah_obd_proxy_episodes, + round( + count(*) FILTER ( + WHERE first_outcome IN ('fail', 'reject', 'abort') + )::numeric / nullif(count(*), 0), + 4 + ) AS nonpass_rate, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY extract(epoch FROM (episode_start-prior_episode_start))/86400.0 + ) AS median_days_since_prior_episode, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY prior_attempt_count + ) AS median_prior_attempts +FROM eligible +GROUP BY GROUPING SETS ( + (), + (extract(year FROM episode_start)::integer) +) +ORDER BY target_year NULLS FIRST; + +ROLLBACK; diff --git a/sql/11_outcome_mapping_audit.sql b/sql/11_outcome_mapping_audit.sql new file mode 100644 index 0000000..f9c5d40 --- /dev/null +++ b/sql/11_outcome_mapping_audit.sql @@ -0,0 +1,114 @@ +-- Aggregate-only audit of source-specific inspection result encodings. +-- +-- Values are controlled inspection categories, never identifiers. Counts below +-- 100 are suppressed so accidental free-text anomalies cannot be surfaced. +BEGIN TRANSACTION READ ONLY; + +WITH normalized AS MATERIALIZED ( + SELECT + lower(coalesce(nullif(btrim(county), ''), '')) AS source_era, + upper(coalesce(nullif(btrim(overall_result), ''), '')) + AS overall_result, + upper(coalesce(nullif(btrim(obd_result), ''), '')) + AS obd_result, + lower(coalesce(nullif(btrim(program_type), ''), '')) + AS program_type, + upper(coalesce(nullif(btrim(test_type), ''), '')) + AS test_type + FROM data.inspection_search + WHERE test_start >= timestamp '2010-01-01' +), +overall_counts AS ( + SELECT + 'source_overall_result'::text AS audit_section, + source_era, + overall_result AS value_1, + CASE + WHEN overall_result IN ('PASS', 'P') THEN 'pass' + WHEN overall_result IN ('FAIL', 'F') THEN 'fail' + WHEN overall_result = 'REJECT' THEN 'reject' + WHEN overall_result = 'ABORT' THEN 'abort' + ELSE '' + END::text AS value_2, + NULL::text AS value_3, + count(*) AS records + FROM normalized + GROUP BY 1, 2, 3, 4 + HAVING count(*) >= 100 +), +utah_unrecognized_context AS ( + SELECT + 'utah_unrecognized_context'::text AS audit_section, + source_era, + overall_result AS value_1, + obd_result AS value_2, + program_type || ' / ' || test_type AS value_3, + count(*) AS records + FROM normalized + WHERE source_era = 'utah' + AND overall_result NOT IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT') + GROUP BY 1, 2, 3, 4, 5 + HAVING count(*) >= 100 +), +overall_obd_crosscheck AS ( + SELECT + 'recognized_overall_vs_obd'::text AS audit_section, + source_era, + overall_result AS value_1, + obd_result AS value_2, + program_type || ' / ' || test_type AS value_3, + count(*) AS records + FROM normalized + WHERE overall_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT') + AND obd_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT') + GROUP BY 1, 2, 3, 4, 5 + HAVING count(*) >= 100 +), +binary_crosscheck AS ( + SELECT + 'binary_overall_vs_obd'::text AS audit_section, + source_era, + CASE + WHEN (overall_result IN ('PASS', 'P')) + = (obd_result IN ('PASS', 'P')) + THEN 'agree' + ELSE 'disagree' + END::text AS value_1, + NULL::text AS value_2, + NULL::text AS value_3, + count(*) AS records + FROM normalized + WHERE overall_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT') + AND obd_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT') + GROUP BY 1, 2, 3 + HAVING count(*) >= 100 +), +ranked AS ( + SELECT + *, + row_number() OVER ( + PARTITION BY audit_section, source_era + ORDER BY records DESC, value_1, value_2, value_3 + ) AS frequency_rank + FROM ( + SELECT * FROM overall_counts + UNION ALL + SELECT * FROM utah_unrecognized_context + UNION ALL + SELECT * FROM overall_obd_crosscheck + UNION ALL + SELECT * FROM binary_crosscheck + ) +) +SELECT + audit_section, + source_era, + value_1, + value_2, + value_3, + records +FROM ranked +WHERE frequency_rank <= 50 +ORDER BY audit_section, source_era, records DESC; + +ROLLBACK; diff --git a/sql/local/20_stage_events.sql b/sql/local/20_stage_events.sql new file mode 100644 index 0000000..45ce78e --- /dev/null +++ b/sql/local/20_stage_events.sql @@ -0,0 +1,152 @@ +-- Normalize explicitly all-VARCHAR CSV staging rows, remove exact duplicate +-- analytical records, and quarantine timestamps whose event order is ambiguous. + +CREATE OR REPLACE TABLE normalized_events AS +SELECT + batch_file, + batch_kind, + batch_start, + batch_end, + try_cast(internal_event_id AS BIGINT) AS internal_event_id, + lower(vehicle_token) AS vehicle_token, + try_cast(vehicle_bucket AS INTEGER) AS vehicle_bucket, + try_cast(event_ts AS TIMESTAMP) AS event_ts, + lower(nullif(trim(source_era), '')) AS source_era, + lower(nullif(trim(public_county), '')) AS public_county, + lower(nullif(trim(canonical_outcome), '')) AS canonical_outcome, + lower(nullif(trim(outcome_label_source), '')) AS outcome_label_source, + lower(nullif(trim(program_type), '')) AS program_type, + upper(nullif(trim(test_type), '')) AS test_type, + upper(nullif(trim(observed_make), '')) AS observed_make, + upper(nullif(trim(observed_model), '')) AS observed_model, + try_cast(observed_model_year AS INTEGER) AS observed_model_year, + CASE + WHEN try_cast(internal_event_id AS BIGINT) IS NULL + THEN 'invalid_internal_event_id' + WHEN vehicle_token IS NULL + OR NOT regexp_full_match(lower(vehicle_token), '^[0-9a-f]{64}$') + THEN 'invalid_vehicle_token' + WHEN try_cast(vehicle_bucket AS INTEGER) IS NULL + OR try_cast(vehicle_bucket AS INTEGER) NOT BETWEEN 0 AND 99 + THEN 'invalid_vehicle_bucket' + WHEN try_cast(event_ts AS TIMESTAMP) IS NULL + THEN 'invalid_event_timestamp' + WHEN try_cast(event_ts AS TIMESTAMP) < TIMESTAMP '2010-01-01' + THEN 'pre_2010_timestamp' + WHEN try_cast(event_ts AS TIMESTAMP) < batch_start + OR try_cast(event_ts AS TIMESTAMP) >= batch_end + THEN 'timestamp_outside_manifest_range' + WHEN lower(nullif(trim(canonical_outcome), '')) IS NOT NULL + AND lower(nullif(trim(canonical_outcome), '')) + NOT IN ('pass', 'fail', 'reject', 'abort') + THEN 'invalid_canonical_outcome' + WHEN lower(nullif(trim(outcome_label_source), '')) IS NOT NULL + AND lower(nullif(trim(outcome_label_source), '')) + NOT IN ('overall_result', 'utah_obd_proxy') + THEN 'invalid_outcome_label_source' + WHEN lower(nullif(trim(canonical_outcome), '')) + IN ('pass', 'fail', 'reject', 'abort') + AND lower(nullif(trim(outcome_label_source), '')) IS NULL + THEN 'missing_outcome_label_source' + WHEN lower(nullif(trim(outcome_label_source), '')) = 'utah_obd_proxy' + AND lower(nullif(trim(source_era), '')) IS DISTINCT FROM 'utah' + THEN 'utah_proxy_non_utah_source' + WHEN lower(nullif(trim(source_era), '')) IN ('slc', 'slco') + AND lower(nullif(trim(public_county), '')) + IS DISTINCT FROM 'salt_lake' + THEN 'inconsistent_public_county' + ELSE NULL + END AS invalid_reason +FROM stg_events; + +CREATE OR REPLACE TABLE ranked_events AS +SELECT + *, + row_number() OVER ( + PARTITION BY + vehicle_token, + vehicle_bucket, + event_ts, + source_era, + public_county, + canonical_outcome, + outcome_label_source, + program_type, + test_type, + observed_make, + observed_model, + observed_model_year + ORDER BY internal_event_id, batch_file + ) AS exact_duplicate_rank +FROM normalized_events; + +CREATE OR REPLACE TABLE conflicting_timestamps AS +SELECT + vehicle_token, + event_ts, + count(*) AS distinct_event_count +FROM ranked_events +WHERE invalid_reason IS NULL + AND exact_duplicate_rank = 1 +GROUP BY vehicle_token, event_ts +HAVING count(*) > 1; + +CREATE OR REPLACE TABLE event_exclusions AS +SELECT + internal_event_id, + vehicle_token, + event_ts, + batch_file, + invalid_reason AS exclusion_reason +FROM ranked_events +WHERE invalid_reason IS NOT NULL + +UNION ALL + +SELECT + internal_event_id, + vehicle_token, + event_ts, + batch_file, + 'exact_duplicate' AS exclusion_reason +FROM ranked_events +WHERE invalid_reason IS NULL + AND exact_duplicate_rank > 1 + +UNION ALL + +SELECT + r.internal_event_id, + r.vehicle_token, + r.event_ts, + r.batch_file, + 'conflicting_same_timestamp' AS exclusion_reason +FROM ranked_events AS r +JOIN conflicting_timestamps AS c + ON c.vehicle_token = r.vehicle_token + AND c.event_ts = r.event_ts +WHERE r.invalid_reason IS NULL + AND r.exact_duplicate_rank = 1; + +CREATE OR REPLACE TABLE clean_events AS +SELECT + r.internal_event_id, + r.vehicle_token, + r.vehicle_bucket, + r.event_ts, + r.source_era, + r.public_county, + r.canonical_outcome, + r.outcome_label_source, + r.program_type, + r.test_type, + r.observed_make, + r.observed_model, + r.observed_model_year +FROM ranked_events AS r +LEFT JOIN conflicting_timestamps AS c + ON c.vehicle_token = r.vehicle_token + AND c.event_ts = r.event_ts +WHERE r.invalid_reason IS NULL + AND r.exact_duplicate_rank = 1 + AND c.vehicle_token IS NULL; diff --git a/sql/local/21_build_episodes.sql b/sql/local/21_build_episodes.sql new file mode 100644 index 0000000..0a14851 --- /dev/null +++ b/sql/local/21_build_episodes.sql @@ -0,0 +1,115 @@ +-- Construct episodes across the complete, globally ordered input history. +-- A gap exactly equal to the configured threshold remains in the same episode. + +CREATE OR REPLACE TABLE sequenced_events AS +WITH with_previous AS ( + SELECT + *, + lag(event_ts) OVER ( + PARTITION BY vehicle_token + ORDER BY event_ts, internal_event_id + ) AS previous_event_ts + FROM clean_events +), +with_markers AS ( + SELECT + *, + CASE + WHEN previous_event_ts IS NULL THEN 1 + WHEN event_ts - previous_event_ts + > (SELECT episode_gap_days FROM build_config) + * INTERVAL '1 day' + THEN 1 + ELSE 0 + END AS starts_episode + FROM with_previous +), +with_episode_number AS ( + SELECT + *, + sum(starts_episode) OVER ( + PARTITION BY vehicle_token + ORDER BY event_ts, internal_event_id + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS episode_number + FROM with_markers +) +SELECT + *, + row_number() OVER ( + PARTITION BY vehicle_token, episode_number + ORDER BY event_ts, internal_event_id + ) AS attempt_number, + row_number() OVER ( + PARTITION BY vehicle_token, episode_number + ORDER BY event_ts DESC, internal_event_id DESC + ) AS reverse_attempt_number +FROM with_episode_number; + +CREATE OR REPLACE TABLE episode_daily_activity AS +SELECT + vehicle_token, + episode_number, + cast(event_ts AS DATE) AS event_date, + count(*) AS events_in_day +FROM sequenced_events +GROUP BY vehicle_token, episode_number, cast(event_ts AS DATE); + +CREATE OR REPLACE TABLE episode_aggregates AS +SELECT + e.vehicle_token, + e.vehicle_bucket, + e.episode_number, + min(e.event_ts) AS episode_start, + max(e.event_ts) AS episode_end, + count(*) AS attempt_count, + count(e.canonical_outcome) AS labeled_attempt_count, + coalesce(bool_or(e.canonical_outcome = 'pass'), false) AS eventually_passed, + max(d.events_in_day) AS max_events_in_day, + (list(e.observed_make ORDER BY e.event_ts DESC, e.internal_event_id DESC) + FILTER (WHERE e.observed_make IS NOT NULL))[1] + AS episode_last_observed_make, + (list(e.observed_model ORDER BY e.event_ts DESC, e.internal_event_id DESC) + FILTER (WHERE e.observed_model IS NOT NULL))[1] + AS episode_last_observed_model, + (list(e.observed_model_year ORDER BY e.event_ts DESC, e.internal_event_id DESC) + FILTER (WHERE e.observed_model_year IS NOT NULL))[1] + AS episode_last_observed_model_year +FROM sequenced_events AS e +JOIN episode_daily_activity AS d + ON d.vehicle_token = e.vehicle_token + AND d.episode_number = e.episode_number + AND d.event_date = cast(e.event_ts AS DATE) +GROUP BY e.vehicle_token, e.vehicle_bucket, e.episode_number; + +CREATE OR REPLACE TABLE inspection_episodes AS +SELECT + a.vehicle_token, + a.vehicle_bucket, + a.episode_number, + a.episode_start, + a.episode_end, + f.internal_event_id AS first_internal_event_id, + f.source_era, + f.public_county, + f.canonical_outcome AS first_outcome, + f.outcome_label_source AS first_outcome_label_source, + f.program_type AS first_program_type, + f.test_type AS first_test_type, + l.canonical_outcome AS final_outcome, + a.attempt_count, + a.labeled_attempt_count, + a.eventually_passed, + a.max_events_in_day, + a.episode_last_observed_make, + a.episode_last_observed_model, + a.episode_last_observed_model_year +FROM episode_aggregates AS a +JOIN sequenced_events AS f + ON f.vehicle_token = a.vehicle_token + AND f.episode_number = a.episode_number + AND f.attempt_number = 1 +JOIN sequenced_events AS l + ON l.vehicle_token = a.vehicle_token + AND l.episode_number = a.episode_number + AND l.reverse_attempt_number = 1; diff --git a/sql/local/22_build_features.sql b/sql/local/22_build_features.sql new file mode 100644 index 0000000..3be93b2 --- /dev/null +++ b/sql/local/22_build_features.sql @@ -0,0 +1,209 @@ +-- Build one explicitly point-in-time row per episode. All predictors carrying +-- history use a window ending at one episode preceding the target. + +CREATE OR REPLACE TABLE episode_history AS +WITH history_windows AS ( + SELECT + e.*, + count(*) OVER prior_all AS prior_episode_count, + coalesce(sum(attempt_count) OVER prior_all, 0) AS prior_total_attempt_count, + count(first_outcome) OVER prior_all AS prior_labeled_episode_count, + lag(attempt_count) OVER by_vehicle AS prior_attempt_count, + lag(first_outcome) OVER by_vehicle AS prior_first_outcome, + lag(final_outcome) OVER by_vehicle AS prior_final_outcome, + lag(episode_start) OVER by_vehicle AS prior_episode_start, + max(CASE + WHEN first_outcome IN ('fail', 'reject', 'abort') + THEN episode_start + END) OVER prior_all AS prior_adverse_episode_start, + count(*) FILTER (WHERE first_outcome = 'pass') OVER prior_all + AS prior_pass_count, + count(*) FILTER (WHERE first_outcome = 'fail') OVER prior_all + AS prior_fail_count, + count(*) FILTER (WHERE first_outcome = 'reject') OVER prior_all + AS prior_reject_count, + count(*) FILTER (WHERE first_outcome = 'abort') OVER prior_all + AS prior_abort_count, + count(*) FILTER ( + WHERE first_outcome IN ('fail', 'reject', 'abort') + ) OVER prior_all AS prior_nonpass_count, + count(first_outcome) OVER prior_three AS last3_labeled_episode_count, + count(*) FILTER ( + WHERE first_outcome IN ('fail', 'reject', 'abort') + ) OVER prior_three AS last3_nonpass_count, + max(max_events_in_day) OVER prior_all AS prior_max_events_in_day, + arg_max(episode_last_observed_make, episode_number) + FILTER (WHERE episode_last_observed_make IS NOT NULL) + OVER prior_all AS last_observed_make, + arg_max(episode_last_observed_model, episode_number) + FILTER (WHERE episode_last_observed_model IS NOT NULL) + OVER prior_all AS last_observed_model, + arg_max(episode_last_observed_model_year, episode_number) + FILTER (WHERE episode_last_observed_model_year IS NOT NULL) + OVER prior_all AS last_observed_model_year + FROM inspection_episodes AS e + WINDOW + by_vehicle AS ( + PARTITION BY vehicle_token + ORDER BY episode_number + ), + prior_all AS ( + PARTITION BY vehicle_token + ORDER BY episode_number + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING + ), + prior_three AS ( + PARTITION BY vehicle_token + ORDER BY episode_number + ROWS BETWEEN 3 PRECEDING AND 1 PRECEDING + ) +) +SELECT + *, + CASE + WHEN prior_episode_start IS NULL THEN NULL + ELSE extract(epoch FROM (episode_start - prior_episode_start)) / 86400.0 + END AS days_since_prior_episode, + CASE + WHEN prior_adverse_episode_start IS NULL THEN NULL + ELSE extract(epoch FROM (episode_start - prior_adverse_episode_start)) + / 86400.0 + END AS days_since_prior_adverse, + prior_nonpass_count::DOUBLE / nullif(prior_labeled_episode_count, 0) + AS prior_nonpass_rate, + prior_pass_count::DOUBLE / nullif(prior_labeled_episode_count, 0) + AS prior_pass_rate, + last3_nonpass_count::DOUBLE / nullif(last3_labeled_episode_count, 0) + AS last3_nonpass_rate +FROM history_windows; + +CREATE OR REPLACE TABLE feature_mart AS +SELECT + vehicle_token, + vehicle_bucket, + vehicle_bucket < 10 AS is_vin_audit, + episode_number::BIGINT AS episode_number, + episode_start, + first_outcome, + first_outcome_label_source AS target_outcome_label_source, + CASE + WHEN first_outcome = 'pass' THEN 0 + WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1 + ELSE NULL + END::INTEGER AS target_nonpass, + episode_start >= TIMESTAMP '2016-01-01' + AND first_outcome IS NOT NULL + AND prior_episode_count >= 1 + AND prior_total_attempt_count <= 50 + AND coalesce(prior_max_events_in_day, 0) <= 4 + AS eligible_returning_target, + CASE + WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context' + WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train' + WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune' + WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate' + WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test' + WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow' + ELSE 'out_of_scope' + END AS temporal_partition, + CASE + WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'before_target_period' + WHEN first_outcome IS NULL THEN 'unlabeled_first_outcome' + WHEN prior_episode_count < 1 THEN 'cold_start' + WHEN prior_total_attempt_count > 50 THEN 'prior_event_count_over_50' + WHEN coalesce(prior_max_events_in_day, 0) > 4 + THEN 'prior_daily_activity_over_4' + ELSE NULL + END AS eligibility_exclusion_reason, + public_county, + source_era, + CASE + WHEN month(episode_start) IN (12, 1, 2) THEN 'winter' + WHEN month(episode_start) IN (3, 4, 5) THEN 'spring' + WHEN month(episode_start) IN (6, 7, 8) THEN 'summer' + ELSE 'fall' + END AS target_season, + CASE + WHEN last_observed_model_year BETWEEN 1886 AND year(episode_start) + 1 + THEN greatest(year(episode_start) - last_observed_model_year, 0) + ELSE NULL + END::INTEGER AS vehicle_age, + prior_episode_count, + prior_total_attempt_count::BIGINT AS prior_total_attempt_count, + prior_attempt_count, + days_since_prior_episode, + days_since_prior_adverse, + prior_nonpass_rate, + prior_first_outcome, + prior_final_outcome, + last_observed_make, + last_observed_model, + last_observed_model_year, + prior_labeled_episode_count, + prior_pass_count, + prior_fail_count, + prior_reject_count, + prior_abort_count, + prior_nonpass_count, + prior_pass_rate, + last3_labeled_episode_count, + last3_nonpass_count, + last3_nonpass_rate, + prior_max_events_in_day, + prior_first_outcome IS NULL AS prior_first_outcome_missing, + last_observed_model_year IS NULL AS prior_model_year_missing, + (SELECT source_data_kind FROM build_config) AS source_data_kind, + (SELECT population_estimate_allowed FROM build_config) + AS population_estimate_allowed +FROM episode_history; + +CREATE OR REPLACE TABLE cohort_flow AS +SELECT + 'event' AS grain, + 'input' AS stage, + 'all_rows' AS reason, + count(*) AS observation_count, + count(DISTINCT try_cast(vehicle_token AS VARCHAR)) AS vehicle_count +FROM stg_events + +UNION ALL + +SELECT + 'event', + 'excluded', + exclusion_reason, + count(*), + count(DISTINCT vehicle_token) +FROM event_exclusions +GROUP BY exclusion_reason + +UNION ALL + +SELECT + 'event', + 'accepted', + 'clean_events', + count(*), + count(DISTINCT vehicle_token) +FROM clean_events + +UNION ALL + +SELECT + 'episode', + 'constructed', + 'all_episodes', + count(*), + count(DISTINCT vehicle_token) +FROM inspection_episodes + +UNION ALL + +SELECT + 'episode', + 'eligible_returning_target', + coalesce(eligibility_exclusion_reason, 'eligible'), + count(*), + count(DISTINCT vehicle_token) +FROM feature_mart +GROUP BY coalesce(eligibility_exclusion_reason, 'eligible'); diff --git a/sql/local/23_validate_mart.sql b/sql/local/23_validate_mart.sql new file mode 100644 index 0000000..05a9640 --- /dev/null +++ b/sql/local/23_validate_mart.sql @@ -0,0 +1,155 @@ +-- Every row reports a count of invariant violations. The Python driver refuses +-- to publish the mart unless every count is zero. + +CREATE OR REPLACE TABLE mart_validation AS +SELECT + 'clean_internal_event_id_unique' AS check_name, + count(*) AS violation_count +FROM ( + SELECT internal_event_id + FROM clean_events + GROUP BY internal_event_id + HAVING count(*) > 1 +) + +UNION ALL + +SELECT + 'episode_key_unique', + count(*) +FROM ( + SELECT vehicle_token, episode_number + FROM inspection_episodes + GROUP BY vehicle_token, episode_number + HAVING count(*) > 1 +) + +UNION ALL + +SELECT + 'mart_key_unique', + count(*) +FROM ( + SELECT vehicle_token, episode_number + FROM feature_mart + GROUP BY vehicle_token, episode_number + HAVING count(*) > 1 +) + +UNION ALL + +SELECT + 'episode_attempts_reconcile', + count(*) +FROM ( + SELECT + e.vehicle_token, + e.episode_number + FROM inspection_episodes AS e + JOIN ( + SELECT vehicle_token, episode_number, count(*) AS actual_attempts + FROM sequenced_events + GROUP BY vehicle_token, episode_number + ) AS a USING (vehicle_token, episode_number) + WHERE e.attempt_count <> a.actual_attempts +) + +UNION ALL + +SELECT + 'episode_gap_strictly_greater_than_threshold', + count(*) +FROM ( + SELECT + episode_start, + lag(episode_end) OVER ( + PARTITION BY vehicle_token ORDER BY episode_number + ) AS prior_episode_end + FROM inspection_episodes +) AS gaps +WHERE prior_episode_end IS NOT NULL + AND episode_start - prior_episode_end + <= (SELECT episode_gap_days FROM build_config) * INTERVAL '1 day' + +UNION ALL + +SELECT + 'prior_episode_count_point_in_time', + count(*) +FROM feature_mart +WHERE prior_episode_count <> episode_number - 1 + +UNION ALL + +SELECT + 'target_mapping_consistent', + count(*) +FROM feature_mart +WHERE target_nonpass IS DISTINCT FROM CASE + WHEN first_outcome = 'pass' THEN 0 + WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1 + ELSE NULL +END + +UNION ALL + +SELECT + 'target_outcome_label_source_consistent', + count(*) +FROM feature_mart +WHERE (first_outcome IS NOT NULL AND ( + target_outcome_label_source IS NULL + OR target_outcome_label_source + NOT IN ('overall_result', 'utah_obd_proxy') + )) + OR (target_outcome_label_source = 'utah_obd_proxy' + AND source_era IS DISTINCT FROM 'utah') + +UNION ALL + +SELECT + 'raw_obd_result_absent_from_mart', + count(*) +FROM information_schema.columns +WHERE table_schema = 'main' + AND table_name = 'feature_mart' + AND lower(column_name) = 'obd_result' + +UNION ALL + +SELECT + 'audit_bucket_consistent', + count(*) +FROM feature_mart +WHERE vehicle_bucket NOT BETWEEN 0 AND 99 + OR is_vin_audit IS DISTINCT FROM (vehicle_bucket < 10) + +UNION ALL + +SELECT + 'eligibility_consistent', + count(*) +FROM feature_mart +WHERE eligible_returning_target IS DISTINCT FROM ( + episode_start >= TIMESTAMP '2016-01-01' + AND first_outcome IS NOT NULL + AND prior_episode_count >= 1 + AND prior_total_attempt_count <= 50 + AND coalesce(prior_max_events_in_day, 0) <= 4 +) + +UNION ALL + +SELECT + 'temporal_partition_consistent', + count(*) +FROM feature_mart +WHERE temporal_partition IS DISTINCT FROM CASE + WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context' + WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train' + WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune' + WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate' + WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test' + WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow' + ELSE 'out_of_scope' +END; diff --git a/tests/test_baselines.py b/tests/test_baselines.py new file mode 100644 index 0000000..ac6b106 --- /dev/null +++ b/tests/test_baselines.py @@ -0,0 +1,545 @@ +"""Tests for the private, leakage-safe baseline training contract.""" + +from __future__ import annotations + +import csv +import gzip +import importlib.util +import sys +import tempfile +import unittest +import warnings +from pathlib import Path +from typing import Dict, Iterable, List, Mapping +from unittest import mock + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = PROJECT_ROOT / "scripts/train_baselines.py" +SPEC = importlib.util.spec_from_file_location("train_baselines", MODULE_PATH) +if SPEC is None or SPEC.loader is None: # pragma: no cover - import machinery guard + raise RuntimeError("Could not load scripts/train_baselines.py") +baselines = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = baselines +SPEC.loader.exec_module(baselines) + + +SKLEARN_AVAILABLE = importlib.util.find_spec("numpy") is not None and importlib.util.find_spec( + "sklearn" +) is not None +DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None + + +def valid_record( + episode_start: str = "2020-06-01T09:00:00", + partition: str = "train", + target: int = 0, + bucket: int = 25, + token_number: int = 1, +) -> Dict[str, object]: + return { + "vehicle_token": "{:064x}".format(token_number), + "vehicle_bucket": bucket, + "is_vin_audit": "true" if bucket < 10 else "false", + "episode_number": 2, + "episode_start": episode_start, + "first_outcome": "fail" if target else "pass", + "target_nonpass": target, + "eligible_returning_target": "true", + "temporal_partition": partition, + "vehicle_age": 10, + "prior_episode_count": 1, + "prior_total_attempt_count": 1, + "prior_attempt_count": 1, + "days_since_prior_episode": 365, + "days_since_prior_adverse": "", + "prior_nonpass_rate": 0, + "public_county": "salt_lake", + "source_era": "slc", + "target_outcome_label_source": "overall_result", + "target_season": "summer", + "prior_first_outcome": "pass", + "prior_final_outcome": "pass", + "last_observed_make": "toyota", + "last_observed_model": "camry", + } + + +def write_csv(path: Path, records: Iterable[Mapping[str, object]]) -> None: + rows = list(records) + if not rows: + raise ValueError("Tests require at least one record") + opener = gzip.open if path.name.endswith(".gz") else open + with opener(path, "wt", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +class SchemaTests(unittest.TestCase): + def test_required_schema_is_accepted(self) -> None: + baselines.validate_schema(list(valid_record().keys())) + + def test_missing_required_column_fails_closed(self) -> None: + columns = list(valid_record().keys()) + columns.remove("prior_first_outcome") + with self.assertRaisesRegex(baselines.SchemaError, "prior_first_outcome"): + baselines.validate_schema(columns) + + def test_target_label_source_column_is_required(self) -> None: + columns = list(valid_record().keys()) + columns.remove("target_outcome_label_source") + with self.assertRaisesRegex( + baselines.SchemaError, "target_outcome_label_source" + ): + baselines.validate_schema(columns) + + def test_forbidden_column_fails_even_when_not_a_feature(self) -> None: + columns = list(valid_record().keys()) + ["vin"] + with self.assertRaisesRegex(baselines.SchemaError, "forbidden"): + baselines.validate_schema(columns) + + def test_unknown_sensitive_column_fails_closed(self) -> None: + columns = list(valid_record().keys()) + ["owner_zip_code"] + with self.assertRaisesRegex(baselines.SchemaError, "owner_zip_code"): + baselines.validate_schema(columns) + + def test_duplicate_columns_fail_closed(self) -> None: + columns = list(valid_record().keys()) + ["vehicle_token"] + with self.assertRaisesRegex(baselines.SchemaError, "duplicate"): + baselines.validate_schema(columns) + + +class MartValidationTests(unittest.TestCase): + def test_loads_csv_gzip_and_assigns_fixed_partitions(self) -> None: + records = [ + valid_record("2022-12-31T23:59:59", "train", 0, 25, 1), + valid_record("2023-01-01T00:00:00", "tune", 1, 25, 2), + valid_record("2024-01-01T00:00:00", "calibrate", 0, 5, 3), + valid_record("2025-01-01T00:00:00", "test", 1, 25, 4), + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "mart.csv.gz" + write_csv(path, records) + mart = baselines.load_mart(path) + self.assertEqual(mart.eligible_rows, 4) + self.assertEqual(len(mart.rows_by_partition["train"]), 1) + self.assertEqual(len(mart.rows_by_partition["tune"]), 1) + self.assertEqual(len(mart.rows_by_partition["calibrate"]), 1) + self.assertEqual(len(mart.rows_by_partition["locked_test"]), 1) + self.assertTrue(mart.rows_by_partition["calibrate"][0].audit_vehicle) + + @unittest.skipUnless(DUCKDB_AVAILABLE, "duckdb is not installed") + def test_loads_parquet_through_duckdb_without_pyarrow(self) -> None: + import duckdb + + with tempfile.TemporaryDirectory() as directory: + csv_path = Path(directory) / "mart.csv" + parquet_path = Path(directory) / "mart.parquet" + write_csv(csv_path, [valid_record()]) + connection = duckdb.connect(database=":memory:") + try: + connection.execute( + "CREATE TABLE mart AS SELECT * FROM read_csv_auto(?)", [str(csv_path)] + ) + escaped_path = str(parquet_path).replace("'", "''") + connection.execute( + "COPY mart TO '{}' (FORMAT PARQUET)".format(escaped_path) + ) + finally: + connection.close() + mart = baselines.load_mart(parquet_path) + self.assertEqual(mart.eligible_rows, 1) + self.assertEqual(len(mart.rows_by_partition["train"]), 1) + + def test_declared_partition_must_match_timestamp(self) -> None: + record = valid_record("2025-05-01T00:00:00", "train") + with self.assertRaisesRegex(baselines.DataValidationError, "belongs to locked_test"): + baselines.episode_from_record(record, 2) + + def test_label_column_must_match_canonical_outcome(self) -> None: + record = valid_record(target=1) + record["target_nonpass"] = 0 + with self.assertRaisesRegex(baselines.DataValidationError, "disagrees"): + baselines.episode_from_record(record, 2) + + def test_audit_boolean_must_match_bucket_contract(self) -> None: + record = valid_record(bucket=5) + record["is_vin_audit"] = "false" + with self.assertRaisesRegex(baselines.DataValidationError, "bucket<10"): + baselines.episode_from_record(record, 2) + + def test_source_era_is_required_for_audit_but_not_modeling(self) -> None: + record = valid_record() + record["source_era"] = "" + with self.assertRaisesRegex(baselines.DataValidationError, "drift auditing"): + baselines.episode_from_record(record, 2) + + def test_utah_proxy_is_accepted_only_for_utah_source_era(self) -> None: + record = valid_record() + record["source_era"] = "utah" + record["target_outcome_label_source"] = "utah_obd_proxy" + episode = baselines.episode_from_record(record, 2) + self.assertEqual(episode.target_outcome_label_source, "utah_obd_proxy") + + record["source_era"] = "slc" + with self.assertRaisesRegex(baselines.DataValidationError, "outside source_era"): + baselines.episode_from_record(record, 2) + + record["source_era"] = "utah" + record["first_outcome"] = "reject" + record["target_nonpass"] = 1 + rejected = baselines.episode_from_record(record, 2) + self.assertEqual(rejected.target, 1) + self.assertEqual(rejected.target_outcome_label_source, "utah_obd_proxy") + + record["first_outcome"] = "abort" + aborted = baselines.episode_from_record(record, 2) + self.assertEqual(aborted.target, 1) + self.assertEqual(aborted.target_outcome_label_source, "utah_obd_proxy") + + def test_unapproved_target_label_source_fails_closed(self) -> None: + record = valid_record() + record["target_outcome_label_source"] = "inferred_four_class" + with self.assertRaisesRegex(baselines.DataValidationError, "unapproved"): + baselines.episode_from_record(record, 2) + + def test_first_episode_cannot_be_eligible_returning_target(self) -> None: + record = valid_record() + record["episode_number"] = 1 + with self.assertRaisesRegex(baselines.DataValidationError, "not a returning"): + baselines.episode_from_record(record, 2) + + def test_ineligible_row_is_not_parsed_as_a_supervised_target(self) -> None: + record = valid_record() + record["eligible_returning_target"] = "false" + record["first_outcome"] = "" + self.assertIsNone(baselines.episode_from_record(record, 2)) + + def test_private_output_path_guard(self) -> None: + accepted = PROJECT_ROOT / "artifacts/private/baselines" + rejected = PROJECT_ROOT / "artifacts/public/baselines" + self.assertEqual(baselines.require_private_path(accepted, "output"), accepted) + with self.assertRaisesRegex(baselines.DataValidationError, "must remain"): + baselines.require_private_path(rejected, "output") + + +class MetricTests(unittest.TestCase): + def test_perfect_ranking_metrics(self) -> None: + targets = [0, 1, 0, 1] + probabilities = [0.1, 0.9, 0.2, 0.8] + metrics = baselines.binary_metrics(targets, probabilities) + self.assertAlmostEqual(metrics["average_precision"], 1.0) + self.assertAlmostEqual(metrics["roc_auc"], 1.0) + self.assertAlmostEqual(metrics["brier"], 0.025) + self.assertAlmostEqual(metrics["top_5_precision"], 1.0) + self.assertAlmostEqual(metrics["top_5_capture"], 0.1) + + def test_capacity_metrics_fractionally_handle_boundary_ties(self) -> None: + targets = [0, 1] * 50 + probabilities = [0.2] * 100 + top_5 = baselines.top_capacity_metrics(targets, probabilities, 0.05) + top_10 = baselines.top_capacity_metrics(targets, probabilities, 0.10) + self.assertAlmostEqual(top_5["precision"], 0.5) + self.assertAlmostEqual(top_5["capture"], 0.05) + self.assertAlmostEqual(top_10["precision"], 0.5) + self.assertAlmostEqual(top_10["capture"], 0.10) + + def test_constant_predictions_have_half_auc(self) -> None: + targets = [0, 1, 0, 1] + probabilities = [0.25] * 4 + self.assertAlmostEqual(baselines.roc_auc(targets, probabilities), 0.5) + self.assertAlmostEqual(baselines.average_precision(targets, probabilities), 0.5) + + def test_calibration_bins_cover_each_row_once(self) -> None: + targets = [0, 1, 0, 1, 1] + probabilities = [0.1, 0.2, 0.3, 0.8, 0.9] + bins = baselines.calibration_bins(targets, probabilities, 3) + self.assertEqual(sum(item["count"] for item in bins), len(targets)) + self.assertEqual([item["count"] for item in bins], [2, 2, 1]) + + +class BaselineBehaviorTests(unittest.TestCase): + def test_default_iteration_budget_is_hardened(self) -> None: + args = baselines.parse_args(["--mart", "data/private/example.parquet"]) + self.assertEqual(args.max_iter, 5000) + + def _episode( + self, + prior_outcome: object, + target: int = 0, + bucket: int = 25, + episode_start: str = "2020-06-01T09:00:00", + partition: str = "train", + source_era: str = "slc", + label_source: str = "overall_result", + ): + record = valid_record( + episode_start=episode_start, + partition=partition, + target=target, + bucket=bucket, + ) + record["prior_first_outcome"] = prior_outcome + record["source_era"] = source_era + record["target_outcome_label_source"] = label_source + return baselines.episode_from_record(record, 2) + + def test_literal_previous_outcome_and_missing_fallback(self) -> None: + rows = [ + self._episode("pass"), + self._episode("fail"), + self._episode("reject"), + self._episode(""), + ] + probabilities, fallback_count = baselines.literal_previous_probabilities(rows, 0.2) + self.assertEqual(probabilities, [0.0, 1.0, 1.0, 0.2]) + self.assertEqual(fallback_count, 1) + + def test_audit_rows_are_excluded_from_fit_cohort(self) -> None: + rows = [self._episode("pass", bucket=5), self._episode("pass", bucket=25)] + non_audit = baselines.evaluation_cohorts(rows, include_audit_breakout=False) + self.assertEqual(non_audit[0][0], "non_audit") + self.assertEqual(len(non_audit[0][1]), 1) + self.assertFalse(non_audit[0][1][0].audit_vehicle) + + def test_locked_manifest_counts_withhold_labels_until_unlocked(self) -> None: + locked = self._episode( + "pass", + target=1, + bucket=25, + episode_start="2025-06-01T09:00:00", + partition="test", + ) + locked_proxy = self._episode( + "pass", + target=0, + bucket=26, + episode_start="2025-06-01T09:00:00", + partition="test", + source_era="utah", + label_source="utah_obd_proxy", + ) + mart = baselines.MartData( + rows_by_partition={ + "train": [], + "tune": [], + "calibrate": [], + "locked_test": [locked, locked_proxy], + }, + input_rows=2, + ineligible_rows=0, + eligible_rows=2, + ) + closed, closed_eras, closed_labels = baselines.manifest_audit_counts( + mart, evaluate_locked=False + ) + self.assertTrue(closed["locked_test"]["labels_withheld"]) + self.assertNotIn("nonpass", closed["locked_test"]) + self.assertNotIn("nonpass", closed_eras["locked_test"]["slc"]) + self.assertEqual( + closed_labels["locked_test"], + {"overall_result": 1, "utah_obd_proxy": 1}, + ) + + opened, opened_eras, opened_labels = baselines.manifest_audit_counts( + mart, evaluate_locked=True + ) + self.assertEqual(opened["locked_test"]["nonpass"], 1) + self.assertEqual(opened_eras["locked_test"]["slc"]["nonpass"], 1) + self.assertEqual( + opened_labels["locked_test"], + {"overall_result": 1, "utah_obd_proxy": 1}, + ) + self.assertFalse( + baselines.TARGET_CONTRACT["utah_obd_proxy_four_class_approved"] + ) + + +@unittest.skipUnless(SKLEARN_AVAILABLE, "numpy/scikit-learn are not installed") +class ModelingIntegrationTests(unittest.TestCase): + @staticmethod + def _partition_records( + timestamp: str, + partition: str, + start_token: int, + future_category: bool = False, + ) -> List[Mapping[str, object]]: + records: List[Mapping[str, object]] = [] + for index in range(40): + target = 1 if index % 4 == 0 else 0 + bucket = index % 100 + record = valid_record( + episode_start=timestamp, + partition=partition, + target=target, + bucket=bucket, + token_number=start_token + index, + ) + record["vehicle_age"] = 2 + index % 25 + record["prior_episode_count"] = 1 + index % 6 + record["prior_total_attempt_count"] = 1 + index % 10 + record["prior_attempt_count"] = 1 + index % 3 + record["days_since_prior_episode"] = 250 + index * 4 + record["days_since_prior_adverse"] = "" if index % 3 else 500 + index + record["prior_nonpass_rate"] = (index % 4) / 4 + record["prior_first_outcome"] = "fail" if index % 5 == 0 else "pass" + if future_category: + record["public_county"] = "future_only_county" + records.append(record) + return records + + def _synthetic_mart(self): + records: List[Mapping[str, object]] = [] + records.extend(self._partition_records("2022-06-01", "train", 1)) + records.extend(self._partition_records("2023-06-01", "tune", 101)) + records.extend( + self._partition_records( + "2024-06-01", "calibrate", 201, future_category=True + ) + ) + records.extend(self._partition_records("2025-06-01", "test", 301)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "mart.csv" + write_csv(path, records) + return baselines.load_mart(path) + + def test_train_only_preprocessing_and_locked_evaluation_gate(self) -> None: + mart = self._synthetic_mart() + trained = baselines.train_baselines( + mart, + c_grid=[0.1], + min_category_count=1, + seed=7, + max_iter=2000, + ) + + feature_names = set(trained.encoder.vectorizer.get_feature_names_out()) + self.assertNotIn("cat__public_county=future_only_county", feature_names) + self.assertFalse(any("source_era" in name for name in feature_names)) + self.assertFalse( + any("target_outcome_label_source" in name for name in feature_names) + ) + + locked_metrics, _ = baselines.evaluate_models( + mart, trained, evaluate_locked=False, bin_count=5 + ) + self.assertNotIn("locked_test", {row["partition"] for row in locked_metrics}) + + unlocked_metrics, _ = baselines.evaluate_models( + mart, trained, evaluate_locked=True, bin_count=5 + ) + locked_rows = [ + row for row in unlocked_metrics if row["partition"] == "locked_test" + ] + self.assertTrue(locked_rows) + self.assertIn("never_fit_audit", {row["cohort"] for row in locked_rows}) + self.assertIn("logistic_platt", {row["model"] for row in locked_rows}) + + tuning = trained.tuning_results[0] + self.assertTrue(tuning["converged"]) + self.assertTrue(tuning["finite_coefficients"]) + self.assertTrue(tuning["finite_probabilities"]) + self.assertTrue(tuning["eligible_for_selection"]) + self.assertLess(tuning["n_iter"], trained.max_iter) + self.assertLess(trained.selected_n_iter, trained.max_iter) + self.assertLess(trained.platt_n_iter, trained.max_iter) + self.assertEqual(trained.platt_model.solver, "liblinear") + self.assertEqual(trained.platt_model.penalty, "l2") + self.assertEqual(trained.platt_model.C, 1000.0) + self.assertEqual( + baselines.PLATT_CALIBRATION_CONFIG, + { + "method": "sigmoid_platt_scaling", + "solver": "liblinear", + "penalty": "l2", + "c": 1000.0, + }, + ) + + def test_deliberately_tiny_max_iter_rejects_all_candidates(self) -> None: + mart = self._synthetic_mart() + with self.assertRaisesRegex( + baselines.DataValidationError, "No logistic candidate converged" + ): + baselines.train_baselines( + mart, + c_grid=[0.03, 0.1], + min_category_count=1, + seed=7, + max_iter=1, + ) + + def test_nonconverged_and_nonfinite_candidates_are_excluded(self) -> None: + import numpy as np + import sklearn + from sklearn.exceptions import ConvergenceWarning + from sklearn.feature_extraction import DictVectorizer + + class ControlledLogistic: + def __init__(self, C=1.0, solver="lbfgs", max_iter=100, **_kwargs): + self.C = C + self.solver = solver + self.max_iter = max_iter + + def fit(self, features, _targets): + self.n_iter_ = np.asarray([2]) + self.coef_ = np.zeros((1, features.shape[1])) + self.intercept_ = np.zeros(1) + if self.solver == "saga" and self.C == 0.03: + warnings.warn("controlled nonconvergence", ConvergenceWarning) + if self.solver == "saga" and self.C == 0.1: + self.coef_[0, 0] = np.nan + return self + + def predict_proba(self, features): + probability = np.full(features.shape[0], 0.25) + return np.column_stack((1.0 - probability, probability)) + + def decision_function(self, features): + return np.zeros(features.shape[0]) + + dependencies = ( + np, + sklearn, + DictVectorizer, + ControlledLogistic, + ConvergenceWarning, + ) + mart = self._synthetic_mart() + with mock.patch.object( + baselines, "require_ml_dependencies", return_value=dependencies + ): + trained = baselines.train_baselines( + mart, + c_grid=[0.03, 0.1, 1.0], + min_category_count=1, + seed=7, + max_iter=100, + ) + + self.assertEqual(trained.selected_c, 1.0) + by_c = {result["c"]: result for result in trained.tuning_results} + self.assertFalse(by_c[0.03]["converged"]) + self.assertFalse(by_c[0.03]["eligible_for_selection"]) + self.assertFalse(by_c[0.1]["finite_coefficients"]) + self.assertFalse(by_c[0.1]["eligible_for_selection"]) + self.assertTrue(by_c[1.0]["eligible_for_selection"]) + + def test_convergence_capture_replays_unrelated_warnings(self) -> None: + from sklearn.exceptions import ConvergenceWarning + + class WarningModel: + def fit(self, _features, _targets): + warnings.warn("unrelated diagnostic", RuntimeWarning) + warnings.warn("did not converge", ConvergenceWarning) + + with self.assertWarnsRegex(RuntimeWarning, "unrelated diagnostic"): + messages = baselines.fit_with_convergence_capture( + WarningModel(), None, None, ConvergenceWarning + ) + self.assertEqual(messages, ["did not converge"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dashboard_export.py b/tests/test_dashboard_export.py new file mode 100644 index 0000000..9cb8dee --- /dev/null +++ b/tests/test_dashboard_export.py @@ -0,0 +1,564 @@ +from __future__ import annotations + +import hashlib +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import duckdb + +from scripts import export_dashboard_data as dashboard_export + + +class DashboardExportTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.mart = self.root / "private_feature_mart.parquet" + self.mart_manifest = self.root / "private_feature_mart.parquet.manifest.json" + self.model_manifest = self.root / "model_manifest.json" + self.model_metrics = self.root / "model_metrics.json" + self.public_root = (self.root / "dashboard/public/data").resolve() + self._write_mart() + self._write_mart_manifest() + self._write_model_files() + + def tearDown(self) -> None: + self.temporary.cleanup() + + def test_development_mart_is_refused_without_preview_flag(self) -> None: + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + with self.assertRaises(dashboard_export.ManifestError): + self._export(development_preview=False) + self.assertFalse(self.public_root.exists()) + + def test_preview_is_suppressed_deterministic_and_contains_no_locked_data(self) -> None: + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + paths = self._export(development_preview=True) + + expected_names = set(dashboard_export.PUBLIC_ASSET_NAMES) | { + dashboard_export.SHA256_MANIFEST_NAME + } + self.assertEqual(set(paths), expected_names) + + first_bytes = { + name: (self.public_root / name).read_bytes() + for name in expected_names + } + joined = b"\n".join(first_bytes.values()).lower() + for forbidden in ( + b"lockedmake", + b"secret_locked", + b"vehicle_token", + b"internal_event_id", + b'"vin"', + b'"plate"', + b'"zip"', + b'"station"', + b"episode_start", + b"target_outcome_label_source", + b'"prediction"', + ): + self.assertNotIn(forbidden, joined) + + for name in expected_names: + payload = json.loads((self.public_root / name).read_text(encoding="utf-8")) + self.assertIs(payload["development_preview"], True) + self.assertIs(payload["population_estimate_allowed"], False) + + scorecards = self._asset("cohort_scorecard.json")["rows"] + self.assertEqual( + scorecards, + [ + { + "prior_make": "SAFE", + "prior_model": "SUPPORTED", + "support_rounded": 400, + "nonpass_rate": 0.167, + } + ], + ) + overview = self._asset("overview_period_county.json")["rows"] + self.assertEqual(len(overview), 3) + self.assertEqual(overview[0]["support_rounded"], 800) + self.assertNotIn("n", overview[0]) + self.assertNotIn("nonpass_count", overview[0]) + + diagnostics = self._asset("model_diagnostics.json")["rows"] + self.assertEqual( + {row["partition"] for row in diagnostics}, + {"train", "tune", "calibrate"}, + ) + self.assertEqual({row["model"] for row in diagnostics}, {"test_model"}) + self.assertTrue(all("episodes" not in row for row in diagnostics)) + + data_manifest = self._asset("data_manifest.json") + self.assertEqual(data_manifest["model_versions"], ["test_model_v1"]) + self.assertRegex(data_manifest["release_id"], r"^[0-9a-f]{64}$") + + checksum_manifest = self._asset(dashboard_export.SHA256_MANIFEST_NAME) + for entry in checksum_manifest["files"]: + content = (self.public_root / entry["name"]).read_bytes() + self.assertEqual(hashlib.sha256(content).hexdigest(), entry["sha256"]) + + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + self._export(development_preview=True, overwrite=True) + second_bytes = { + name: (self.public_root / name).read_bytes() + for name in expected_names + } + self.assertEqual(first_bytes, second_bytes) + + def test_unlocked_model_manifest_is_refused(self) -> None: + manifest = self._read_json(self.model_manifest) + manifest["locked_test_evaluated"] = True + self._write_json(self.model_manifest, manifest) + + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + with self.assertRaises(dashboard_export.ManifestError): + self._export(development_preview=True) + self.assertFalse(self.public_root.exists()) + + def test_locked_metric_row_is_refused_even_when_manifest_is_closed(self) -> None: + metrics = self._read_json(self.model_metrics) + locked = dict(metrics[0]) + locked["partition"] = "locked_test" + metrics.append(locked) + self._write_json(self.model_metrics, metrics) + self._refresh_metrics_digest() + + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + with self.assertRaises(dashboard_export.ManifestError): + self._export(development_preview=True) + self.assertFalse(self.public_root.exists()) + + def test_metrics_digest_mismatch_is_refused(self) -> None: + metrics = self._read_json(self.model_metrics) + metrics[0]["average_precision"] = 0.99 + self._write_json(self.model_metrics, metrics) + + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + with self.assertRaises(dashboard_export.ManifestError): + self._export(development_preview=True) + self.assertFalse(self.public_root.exists()) + + def test_overwrite_refuses_unexpected_file(self) -> None: + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + self._export(development_preview=True) + unexpected = self.public_root / "private-notes.txt" + unexpected.write_text("must not ride along\n", encoding="utf-8") + with self.assertRaises(dashboard_export.PublicSchemaError): + self._export(development_preview=True, overwrite=True) + self.assertTrue(unexpected.exists()) + + def test_overwrite_refuses_directory_in_public_contract(self) -> None: + self.public_root.mkdir(parents=True) + (self.public_root / "data_manifest.json").mkdir() + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + with self.assertRaises(dashboard_export.PublicSchemaError): + self._export(development_preview=True, overwrite=True) + + def test_pre_2025_locked_partition_aliases_are_refused(self) -> None: + for partition in dashboard_export.LOCKED_PARTITION_ALIASES: + with self.subTest(partition=partition): + self._move_locked_rows_before_boundary(partition) + with mock.patch.object( + dashboard_export, "PUBLIC_DATA_ROOT", self.public_root + ): + with self.assertRaises(dashboard_export.ManifestError): + self._export(development_preview=True) + self.assertFalse(self.public_root.exists()) + + def test_public_validator_rejects_sensitive_keys_and_exact_timestamps(self) -> None: + base = { + "schema_version": dashboard_export.SCHEMA_VERSION, + "development_preview": True, + "population_estimate_allowed": False, + } + unsafe_key = { + **base, + "rows": [ + { + "year": 2022, + "quarter": 1, + "public_county": "salt_lake", + "support_rounded": 100, + "nonpass_rate": 0.2, + "vehicle_token": "not-public", + } + ], + } + with self.assertRaises(dashboard_export.PublicSchemaError): + dashboard_export.validate_public_asset( + "overview_period_county.json", unsafe_key + ) + + exact_timestamp = { + **base, + "rows": [ + { + "year": 2022, + "quarter": 1, + "public_county": "2022-01-01T12:34:56", + "support_rounded": 100, + "nonpass_rate": 0.2, + } + ], + } + with self.assertRaises(dashboard_export.PublicSchemaError): + dashboard_export.validate_public_asset( + "overview_period_county.json", exact_timestamp + ) + + def _export( + self, + development_preview: bool, + overwrite: bool = False, + ) -> dict: + return dashboard_export.export_dashboard_data( + mart=self.mart, + mart_manifest=self.mart_manifest, + bundles=[ + dashboard_export.ModelBundle( + manifest=self.model_manifest, + metrics=self.model_metrics, + ) + ], + output_dir=self.public_root, + development_preview=development_preview, + overwrite=overwrite, + ) + + def _write_mart(self) -> None: + connection = duckdb.connect(":memory:") + try: + connection.execute( + """ + CREATE TABLE mart ( + vehicle_token VARCHAR, + is_vin_audit BOOLEAN, + episode_number BIGINT, + episode_start TIMESTAMP, + first_outcome VARCHAR, + target_outcome_label_source VARCHAR, + target_nonpass INTEGER, + eligible_returning_target BOOLEAN, + temporal_partition VARCHAR, + public_county VARCHAR, + source_era VARCHAR, + vehicle_age INTEGER, + last_observed_make VARCHAR, + last_observed_model VARCHAR + ) + """ + ) + rows = [] + rows.extend(self._group(120, 20, "SAFE", "SUPPORTED")) + rows.extend(self._group(99, 19, "LOW", "SUPPORT")) + rows.extend(self._group(120, 9, "LOW", "NONPASS")) + rows.extend(self._group(120, 111, "LOW", "PASS")) + rows.extend( + self._group( + 120, + 60, + "ENTITY", + "LOWTOTAL", + pass_vehicle_count=40, + nonpass_vehicle_count=40, + ) + ) + rows.extend( + self._group( + 120, + 20, + "ENTITY", + "LOWNONPASS", + nonpass_vehicle_count=9, + ) + ) + rows.extend( + self._group( + 120, + 100, + "ENTITY", + "LOWPASS", + pass_vehicle_count=9, + ) + ) + rows.extend( + self._group( + 120, + 20, + "SAFE", + "SUPPORTED", + temporal_partition="tune", + ) + ) + rows.extend( + self._group( + 120, + 20, + "SAFE", + "SUPPORTED", + temporal_partition="calibrate", + ) + ) + for index in range(120): + rows.append( + ( + "secret_vehicle_token", + False, + 2, + "2025-01-15 12:34:56", + "secret_locked", + "secret_locked_source", + 999, + True, + "locked_test", + "locked_county", + "locked_source", + 5, + "LOCKEDMAKE", + "LOCKEDMODEL", + ) + ) + connection.executemany( + "INSERT INTO mart VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + rows, + ) + connection.table("mart").write_parquet(str(self.mart)) + finally: + connection.close() + + @staticmethod + def _group( + n: int, + nonpass: int, + make: str, + model: str, + temporal_partition: str = "train", + pass_vehicle_count: int = None, + nonpass_vehicle_count: int = None, + ) -> list: + partition_dates = { + "train": "2022-01-15 08:00:00", + "tune": "2023-01-15 08:00:00", + "calibrate": "2024-01-15 08:00:00", + } + pass_count = n - nonpass + pass_vehicle_count = pass_count if pass_vehicle_count is None else pass_vehicle_count + nonpass_vehicle_count = ( + nonpass if nonpass_vehicle_count is None else nonpass_vehicle_count + ) + rows = [] + for index in range(n): + adverse = index < nonpass + class_index = index if adverse else index - nonpass + class_vehicle_count = ( + nonpass_vehicle_count if adverse else pass_vehicle_count + ) + outcome = "nonpass" if adverse else "pass" + vehicle_token = "{}_{}_{}_{}_{}".format( + make, + model, + temporal_partition, + outcome, + class_index % class_vehicle_count, + ) + rows.append( + ( + vehicle_token, + False, + 2, + partition_dates[temporal_partition], + "fail" if adverse else "pass", + "overall_result", + 1 if adverse else 0, + True, + temporal_partition, + "salt_lake", + "slc", + 5, + make, + model, + ) + ) + return rows + + def _write_mart_manifest(self) -> None: + connection = duckdb.connect(":memory:") + try: + columns = [ + {"name": row[0], "duckdb_type": row[1]} + for row in connection.execute( + "DESCRIBE SELECT * FROM read_parquet(?)", [str(self.mart)] + ).fetchall() + ] + finally: + connection.close() + self._write_json( + self.mart_manifest, + { + "build_kind": "private_leakage_safe_inspection_feature_mart", + "classification": "private_pseudonymized_analytical_mart", + "source_data_kind": "vehicle_history_development_sample", + "population_estimate_allowed": False, + "episode_gap_days": 30, + "columns": columns, + "parquet_sha256": self._sha256(self.mart), + }, + ) + + def _write_model_files(self) -> None: + connection = duckdb.connect(":memory:") + try: + private_support = { + partition: (episodes, nonpass, vehicles) + for partition, episodes, nonpass, vehicles in connection.execute( + """ + SELECT temporal_partition, + count(*)::BIGINT, + sum(target_nonpass)::BIGINT, + count(DISTINCT vehicle_token)::BIGINT + FROM read_parquet(?) + WHERE temporal_partition IN ('train', 'tune', 'calibrate') + AND eligible_returning_target + AND target_nonpass IN (0, 1) + AND NOT is_vin_audit + GROUP BY 1 + """, + [str(self.mart)], + ).fetchall() + } + finally: + connection.close() + + rows = [] + for partition, value in ( + ("train", 0.25), + ("tune", 0.24), + ("calibrate", 0.23), + ): + episodes, nonpass, vehicles = private_support[partition] + rows.append( + { + "model": "test_model", + "partition": partition, + "cohort": "non_audit", + "episodes": episodes, + "nonpass": nonpass, + "vehicles": vehicles, + "average_precision": value, + "brier": 0.1, + "log_loss": 0.3, + "roc_auc": 0.7, + "top_10_capture": 0.2, + } + ) + for model, episodes, nonpass in ( + ("low_support", 99, 20), + ("low_nonpass", 120, 9), + ("low_pass", 120, 111), + ("low_vehicles", 120, 20), + ): + rows.append( + { + "model": model, + "partition": "train", + "cohort": "non_audit", + "episodes": episodes, + "nonpass": nonpass, + "vehicles": 99 if model == "low_vehicles" else episodes, + "average_precision": 0.2, + "brier": 0.1, + "log_loss": 0.3, + "roc_auc": 0.7, + "top_10_capture": 0.2, + } + ) + self._write_json(self.model_metrics, rows) + self._write_json( + self.model_manifest, + { + "classification": "private_model_artifact_no_row_predictions", + "input_sha256": self._sha256(self.mart), + "metrics_sha256": self._sha256(self.model_metrics), + "locked_test_evaluated": False, + "model_version": "test_model_v1", + }, + ) + + def _refresh_metrics_digest(self) -> None: + manifest = self._read_json(self.model_manifest) + manifest["metrics_sha256"] = self._sha256(self.model_metrics) + self._write_json(self.model_manifest, manifest) + + def _move_locked_rows_before_boundary(self, partition: str) -> None: + rewritten = self.root / "rewritten.parquet" + connection = duckdb.connect(":memory:") + try: + connection.execute( + "CREATE TABLE rewritten AS SELECT * FROM read_parquet(?)", + [str(self.mart)], + ) + connection.execute( + """ + UPDATE rewritten + SET temporal_partition = ?, + episode_start = TIMESTAMP '2024-12-31 12:34:56' + WHERE last_observed_make = 'LOCKEDMAKE' + """, + [partition], + ) + connection.table("rewritten").write_parquet(str(rewritten)) + finally: + connection.close() + rewritten.replace(self.mart) + + mart_manifest = self._read_json(self.mart_manifest) + mart_manifest["parquet_sha256"] = self._sha256(self.mart) + self._write_json(self.mart_manifest, mart_manifest) + + model_manifest = self._read_json(self.model_manifest) + model_manifest["input_sha256"] = self._sha256(self.mart) + self._write_json(self.model_manifest, model_manifest) + + def _asset(self, name: str) -> object: + return self._read_json(self.public_root / name) + + @staticmethod + def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + @staticmethod + def _write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + + @staticmethod + def _read_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_feature_mart.py b/tests/test_feature_mart.py new file mode 100644 index 0000000..4a75b5c --- /dev/null +++ b/tests/test_feature_mart.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +import csv +import gzip +import hashlib +import json +import tempfile +import unittest +from pathlib import Path +from typing import Dict, Iterable, List, Mapping, Optional + +import duckdb + +from scripts import build_feature_mart as mart + + +def token(character: str) -> str: + return character * 64 + + +def bucket(vehicle_token: str) -> int: + return int(vehicle_token[:8], 16) % 100 + + +def event( + event_id: int, + vehicle_token: str, + event_ts: str, + outcome: Optional[str], + make: Optional[str] = "TOYOTA", + model: Optional[str] = "CAMRY", + model_year: Optional[int] = 2010, + source_era: str = "slc", + public_county: str = "salt_lake", + label_source: Optional[str] = "overall_result", +) -> Dict[str, object]: + return { + "internal_event_id": event_id, + "vehicle_token": vehicle_token, + "vehicle_bucket": bucket(vehicle_token), + "event_ts": event_ts, + "source_era": source_era, + "public_county": public_county, + "canonical_outcome": outcome, + "outcome_label_source": label_source if outcome is not None else None, + "program_type": "obd", + "test_type": "OBDII", + "observed_make": make, + "observed_model": model, + "observed_model_year": model_year, + } + + +def write_export( + directory: Path, + name: str, + rows: Iterable[Mapping[str, object]], + start: str, + end: str, + kind: str = mart.DEVELOPMENT_KIND, +) -> Path: + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + materialized = list(rows) + with gzip.open(path, "wt", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=mart.EXPECTED_COLUMNS) + writer.writeheader() + writer.writerows(materialized) + + digest = mart.file_sha256(path) + manifest = { + "export_kind": kind, + "query_version": ( + "inspection_batch_v2" + if kind == mart.BOUNDED_KIND + else "inspection_history_development_sample_v1" + ), + "query_sha256": "1" * 64, + "generated_at_utc": "2026-07-15T12:00:00+00:00", + "source_start_inclusive": start, + "source_end_exclusive": end, + "vehicle_token_key_version": "v1", + "vehicle_token_key_fingerprint": "abcdef1234567890", + "rows": len(materialized), + "columns": list(mart.EXPECTED_COLUMNS), + "compressed_file_sha256": digest, + "compressed_file_bytes": path.stat().st_size, + "classification": ( + "private_pseudonymized_analytical_staging" + if kind == mart.BOUNDED_KIND + else "private_pseudonymized_development_only" + ), + } + if kind == mart.DEVELOPMENT_KIND: + manifest.update( + { + "population_estimate_allowed": False, + "vehicle_sample_limit": 100, + "sample_method": "synthetic_test_fixture", + } + ) + manifest_path = path.with_name(path.name + ".manifest.json") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return path + + +class FeatureMartTest(unittest.TestCase): + def setUp(self) -> None: + self._temporary_directory = tempfile.TemporaryDirectory() + self.root = Path(self._temporary_directory.name) + + def tearDown(self) -> None: + self._temporary_directory.cleanup() + + def build(self, input_path: Path) -> mart.BuildSummary: + return mart.build_feature_mart( + mart.BuildConfig( + input_path=input_path, + database_path=self.root / "warehouse.duckdb", + output_path=self.root / "feature_mart.parquet", + memory_limit="512MB", + threads=1, + allow_external_output=True, + ) + ) + + def row_dict( + self, connection: duckdb.DuckDBPyConnection, sql: str, parameters=None + ) -> Dict[str, object]: + cursor = connection.execute(sql, parameters or []) + names = [item[0] for item in cursor.description] + row = cursor.fetchone() + self.assertIsNotNone(row) + return dict(zip(names, row)) + + def test_episode_boundary_quotes_deduplication_and_prior_only_features(self) -> None: + vehicle = token("a") + rows = [ + event( + 1, + vehicle, + "2015-01-01 00:00:00", + "pass", + make="ACME, MOTORS", + model='MODEL "Q"', + ), + event( + 2, + vehicle, + "2015-01-31 00:00:00", + "fail", + make="ACME, MOTORS", + model='MODEL "Q"', + ), + # Exact analytical duplicate with a distinct source row ID. + event( + 3, + vehicle, + "2015-01-01 00:00:00", + "pass", + make="ACME, MOTORS", + model='MODEL "Q"', + ), + # Thirty days and one second after the preceding accepted event. + event( + 4, + vehicle, + "2015-03-02 00:00:01", + "reject", + make="ACME, MOTORS", + model='MODEL "Q"', + ), + event( + 5, + vehicle, + "2016-03-02 00:00:01", + "fail", + make="TARGET, LEAK", + model="CURRENT ROW", + ), + event( + 6, + vehicle, + "2016-03-03 00:00:01", + "pass", + make="LATER LEAK", + model="LATER ATTEMPT", + ), + ] + source = write_export( + self.root, + "history.csv.gz", + rows, + "2010-01-01T00:00:00", + "2020-01-01T00:00:00", + ) + summary = self.build(source) + self.assertEqual(summary.clean_events, 5) + self.assertEqual(summary.episodes, 3) + self.assertFalse(summary.population_estimate_allowed) + + connection = duckdb.connect(str(summary.database_path), read_only=True) + try: + first_episode = self.row_dict( + connection, + """ + SELECT attempt_count, first_outcome, final_outcome + FROM inspection_episodes + WHERE vehicle_token = ? AND episode_number = 1 + """, + [vehicle], + ) + self.assertEqual(first_episode["attempt_count"], 2) + self.assertEqual(first_episode["first_outcome"], "pass") + self.assertEqual(first_episode["final_outcome"], "fail") + + target = self.row_dict( + connection, + "SELECT * FROM feature_mart WHERE vehicle_token = ? " + "AND episode_number = 3", + [vehicle], + ) + self.assertEqual(target["target_nonpass"], 1) + self.assertEqual( + target["target_outcome_label_source"], "overall_result" + ) + self.assertTrue(target["eligible_returning_target"]) + self.assertEqual(target["prior_episode_count"], 2) + self.assertEqual(target["prior_total_attempt_count"], 3) + self.assertEqual(target["prior_attempt_count"], 1) + self.assertEqual(target["prior_first_outcome"], "reject") + self.assertEqual(target["prior_final_outcome"], "reject") + self.assertEqual(target["last_observed_make"], "ACME, MOTORS") + self.assertEqual(target["last_observed_model"], 'MODEL "Q"') + self.assertEqual(target["vehicle_age"], 6) + self.assertEqual(target["temporal_partition"], "train") + + names = { + row[0] + for row in connection.execute( + "DESCRIBE SELECT * FROM feature_mart" + ).fetchall() + } + required = { + "vehicle_token", + "vehicle_bucket", + "is_vin_audit", + "episode_number", + "episode_start", + "first_outcome", + "target_nonpass", + "eligible_returning_target", + "temporal_partition", + "vehicle_age", + "prior_episode_count", + "prior_total_attempt_count", + "prior_attempt_count", + "days_since_prior_episode", + "days_since_prior_adverse", + "prior_nonpass_rate", + "public_county", + "source_era", + "target_season", + "prior_first_outcome", + "prior_final_outcome", + "last_observed_make", + "last_observed_model", + } + self.assertTrue(required.issubset(names)) + self.assertTrue( + { + "attempt_count", + "final_outcome", + "eventually_passed", + "episode_end", + "obd_result", + }.isdisjoint(names) + ) + finally: + connection.close() + + def test_conflicting_timestamp_is_quarantined_and_null_first_stays_null(self) -> None: + conflicted = token("b") + unlabeled = token("c") + rows = [ + event(10, conflicted, "2014-01-01", "pass"), + event(11, conflicted, "2016-01-01", "pass"), + event(12, conflicted, "2016-01-01", "fail"), + event(13, conflicted, "2017-01-01", "pass"), + event(20, unlabeled, "2015-01-01", "pass"), + event(21, unlabeled, "2016-01-01", None), + event(22, unlabeled, "2016-01-02", "pass"), + ] + source = write_export( + self.root, + "history.csv.gz", + rows, + "2010-01-01", + "2020-01-01", + ) + summary = self.build(source) + connection = duckdb.connect(str(summary.database_path), read_only=True) + try: + conflict_count = connection.execute( + "SELECT count(*) FROM event_exclusions " + "WHERE exclusion_reason = 'conflicting_same_timestamp'" + ).fetchone()[0] + self.assertEqual(conflict_count, 2) + target = self.row_dict( + connection, + "SELECT first_outcome, target_nonpass, " + "eligible_returning_target, eligibility_exclusion_reason " + "FROM feature_mart WHERE vehicle_token = ? " + "AND episode_start = TIMESTAMP '2016-01-01'", + [unlabeled], + ) + self.assertIsNone(target["first_outcome"]) + self.assertIsNone(target["target_nonpass"]) + self.assertFalse(target["eligible_returning_target"]) + self.assertEqual( + target["eligibility_exclusion_reason"], "unlabeled_first_outcome" + ) + finally: + connection.close() + + def test_bounded_files_are_combined_before_episode_construction(self) -> None: + vehicle = token("d") + first = write_export( + self.root, + "inspection_202601.csv.gz", + [event(30, vehicle, "2026-01-31 12:00:00", "pass")], + "2026-01-01", + "2026-02-01", + kind=mart.BOUNDED_KIND, + ) + write_export( + self.root, + "inspection_202602.csv.gz", + [event(31, vehicle, "2026-02-01 12:00:00", "fail")], + "2026-02-01", + "2026-03-01", + kind=mart.BOUNDED_KIND, + ) + summary = self.build(first.parent) + self.assertTrue(summary.population_estimate_allowed) + connection = duckdb.connect(str(summary.database_path), read_only=True) + try: + result = connection.execute( + "SELECT count(*), max(attempt_count) FROM inspection_episodes" + ).fetchone() + self.assertEqual(result, (1, 2)) + finally: + connection.close() + + def test_vehicle_quality_eligibility_uses_only_prior_activity(self) -> None: + vehicle = token("e") + rows: List[Mapping[str, object]] = [ + event(40, vehicle, "2014-01-01 00:00:00", "pass"), + event(41, vehicle, "2016-01-01 00:00:00", "pass"), + ] + rows.extend( + event(42 + index, vehicle, f"2016-03-10 0{index}:00:00", "fail") + for index in range(5) + ) + rows.append(event(47, vehicle, "2017-04-15 00:00:00", "pass")) + source = write_export( + self.root, + "history.csv.gz", + rows, + "2010-01-01", + "2020-01-01", + ) + summary = self.build(source) + connection = duckdb.connect(str(summary.database_path), read_only=True) + try: + current_busy_episode = self.row_dict( + connection, + "SELECT eligible_returning_target, prior_max_events_in_day " + "FROM feature_mart WHERE vehicle_token = ? " + "AND episode_start = TIMESTAMP '2016-03-10 00:00:00'", + [vehicle], + ) + self.assertTrue(current_busy_episode["eligible_returning_target"]) + self.assertEqual(current_busy_episode["prior_max_events_in_day"], 1) + + after_busy_episode = self.row_dict( + connection, + "SELECT eligible_returning_target, prior_max_events_in_day, " + "eligibility_exclusion_reason FROM feature_mart " + "WHERE vehicle_token = ? " + "AND episode_start = TIMESTAMP '2017-04-15 00:00:00'", + [vehicle], + ) + self.assertFalse(after_busy_episode["eligible_returning_target"]) + self.assertEqual(after_busy_episode["prior_max_events_in_day"], 5) + self.assertEqual( + after_busy_episode["eligibility_exclusion_reason"], + "prior_daily_activity_over_4", + ) + finally: + connection.close() + + def test_digest_mismatch_fails_before_build(self) -> None: + source = write_export( + self.root, + "history.csv.gz", + [event(50, token("f"), "2016-01-01", "pass")], + "2010-01-01", + "2020-01-01", + ) + manifest_path = source.with_name(source.name + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["compressed_file_sha256"] = "0" * 64 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with self.assertRaisesRegex(mart.MartBuildError, "SHA-256 mismatch"): + self.build(source) + + def test_outcome_label_source_is_validated_and_first_row_lineage_is_audit_only( + self, + ) -> None: + utah_vehicle = token("7") + rows = [ + event( + 70, + utah_vehicle, + "2015-01-01", + "pass", + source_era="utah", + public_county="utah", + label_source="utah_obd_proxy", + ), + event( + 71, + utah_vehicle, + "2016-01-01", + "fail", + source_era="utah", + public_county="utah", + label_source="utah_obd_proxy", + ), + event( + 72, + token("8"), + "2016-01-01", + "pass", + source_era="slc", + public_county="salt_lake", + label_source="utah_obd_proxy", + ), + event( + 73, + token("9"), + "2016-01-01", + "pass", + label_source=None, + ), + event( + 74, + token("0"), + "2016-01-01", + "pass", + label_source="unknown_source", + ), + ] + source = write_export( + self.root, + "history.csv.gz", + rows, + "2010-01-01", + "2020-01-01", + ) + summary = self.build(source) + connection = duckdb.connect(str(summary.database_path), read_only=True) + try: + target = self.row_dict( + connection, + "SELECT target_outcome_label_source, source_era " + "FROM feature_mart WHERE vehicle_token = ? " + "AND episode_number = 2", + [utah_vehicle], + ) + self.assertEqual( + target["target_outcome_label_source"], "utah_obd_proxy" + ) + self.assertEqual(target["source_era"], "utah") + + reasons = dict( + connection.execute( + "SELECT exclusion_reason, count(*) FROM event_exclusions " + "GROUP BY exclusion_reason" + ).fetchall() + ) + self.assertEqual(reasons["utah_proxy_non_utah_source"], 1) + self.assertEqual(reasons["missing_outcome_label_source"], 1) + self.assertEqual(reasons["invalid_outcome_label_source"], 1) + + schema_names = { + row[0] + for row in connection.execute( + "DESCRIBE SELECT * FROM feature_mart" + ).fetchall() + } + self.assertIn("target_outcome_label_source", schema_names) + self.assertNotIn("outcome_label_source", schema_names) + self.assertNotIn("obd_result", schema_names) + finally: + connection.close() + + def test_bounded_gap_fails_closed(self) -> None: + vehicle = token("1") + write_export( + self.root, + "one.csv.gz", + [event(60, vehicle, "2026-01-15", "pass")], + "2026-01-01", + "2026-02-01", + kind=mart.BOUNDED_KIND, + ) + write_export( + self.root, + "two.csv.gz", + [event(61, vehicle, "2026-03-15", "pass")], + "2026-03-01", + "2026-04-01", + kind=mart.BOUNDED_KIND, + ) + with self.assertRaisesRegex(mart.MartBuildError, "interval gap"): + self.build(self.root) + + incomplete = mart.build_feature_mart( + mart.BuildConfig( + input_path=self.root, + database_path=self.root / "incomplete.duckdb", + output_path=self.root / "incomplete.parquet", + memory_limit="512MB", + threads=1, + allow_gaps=True, + allow_external_output=True, + ) + ) + self.assertFalse(incomplete.population_estimate_allowed) + build_manifest = json.loads( + incomplete.manifest_path.read_text(encoding="utf-8") + ) + self.assertFalse(build_manifest["population_estimate_allowed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_private_export.py b/tests/test_private_export.py new file mode 100644 index 0000000..f22bbdc --- /dev/null +++ b/tests/test_private_export.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import hashlib +import hmac +import os +import tempfile +import unittest +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from scripts import export_history_sample as history_export +from scripts import export_inspection_batch as exporter + + +VALID_VIN = "1ABCD23EFGH456789" +TEST_KEY = bytes(range(32)) + + +class PrivateExportTests(unittest.TestCase): + def test_normalize_vin_canonicalizes_case_and_whitespace(self) -> None: + self.assertEqual( + exporter.normalize_vin(f" {VALID_VIN.lower()}\n"), + VALID_VIN, + ) + + def test_normalize_vin_rejects_invalid_values_and_placeholders(self) -> None: + invalid_vins = ( + "", + "1ABCD23EFGH45678", + "1ABCD23EFGH4567890", + "1ABCD23EFGI456789", + "11111111111111111", + "AAAAAAAAAAAAAAAAA", + "12345678901234567", + "98765432109876543", + ) + for vin in invalid_vins: + with self.subTest(vin=vin): + self.assertIsNone(exporter.normalize_vin(vin)) + + def test_hmac_is_deterministic_and_version_domain_separated(self) -> None: + expected_v1 = hmac.new( + TEST_KEY, + b"utah-vehicle-health/v1/vin\0" + VALID_VIN.encode("ascii"), + hashlib.sha256, + ).hexdigest() + + token_v1 = exporter.vehicle_token(VALID_VIN, TEST_KEY, "v1") + repeated_v1 = exporter.vehicle_token(VALID_VIN, TEST_KEY, "v1") + token_v2 = exporter.vehicle_token(VALID_VIN, TEST_KEY, "v2") + + self.assertEqual(token_v1, expected_v1) + self.assertEqual(repeated_v1, token_v1) + self.assertNotEqual(token_v2, token_v1) + self.assertEqual(len(token_v1), 64) + + def test_recognized_overall_result_precedes_utah_obd_proxy(self) -> None: + self.assertEqual( + exporter.canonicalize_outcome( + "PASS", "FAIL", "utah", "other", "C" + ), + ("pass", "overall_result"), + ) + self.assertEqual( + exporter.canonicalize_outcome( + " f ", "PASS", " UTAH ", "tsi", "TSI" + ), + ("fail", "overall_result"), + ) + + def test_utah_only_obd_fallback_uses_recognized_values(self) -> None: + self.assertEqual( + exporter.canonicalize_outcome( + "", " P ", "utah", " ObD ", " obd " + ), + ("pass", "utah_obd_proxy"), + ) + self.assertEqual( + exporter.canonicalize_outcome( + "B", "REJECT", "UTAH", "obd", "OBD" + ), + ("reject", "utah_obd_proxy"), + ) + self.assertEqual( + exporter.canonicalize_outcome( + None, "FAIL", "weber", "obd", "OBD" + ), + (None, None), + ) + + def test_utah_non_obd_program_or_test_rows_do_not_use_proxy(self) -> None: + excluded_cases = ( + ("other", "C"), + ("tsi", "TSI"), + ("obd", "C"), + ("other", "OBD"), + ("", ""), + (None, None), + ) + for program_type, test_type in excluded_cases: + with self.subTest(program_type=program_type, test_type=test_type): + self.assertEqual( + exporter.canonicalize_outcome( + "B", "PASS", "utah", program_type, test_type + ), + (None, None), + ) + + def test_unknown_blank_and_b_proxy_values_remain_unlabeled(self) -> None: + cases = ( + (None, None, "utah", "obd", "OBD"), + ("", "", "utah", "obd", "OBD"), + ("B", "B", "utah", "obd", "OBD"), + ("UNKNOWN", "PASS", "slco", "obd", "OBD"), + ) + for overall, obd, source, program_type, test_type in cases: + with self.subTest( + overall=overall, + obd=obd, + source=source, + program_type=program_type, + test_type=test_type, + ): + self.assertEqual( + exporter.canonicalize_outcome( + overall, obd, source, program_type, test_type + ), + (None, None), + ) + + def test_hmac_configuration_rejects_malformed_or_weak_keys(self) -> None: + invalid_keys = ( + "", + "not-hex", + "00" * 31, + "00" * 33, + "00" * 32, + "ab" * 32, + ) + for encoded_key in invalid_keys: + with self.subTest(encoded_key=encoded_key[:12]): + with self._hmac_environment(encoded_key): + with self.assertRaises(ValueError): + exporter.get_hmac_configuration() + + def test_hmac_configuration_accepts_exact_random_key(self) -> None: + with self._hmac_environment(TEST_KEY.hex(), version="key-2026.1"): + key, version, fingerprint = exporter.get_hmac_configuration() + + self.assertEqual(key, TEST_KEY) + self.assertEqual(version, "key-2026.1") + self.assertRegex(fingerprint, r"^[0-9a-f]{16}$") + + def test_vehicle_bucket_is_stable_after_vin_normalization(self) -> None: + row = self._source_row(VALID_VIN) + normalized_row = exporter.private_row(row, TEST_KEY, "v1") + lower_row = exporter.private_row( + self._source_row(f" {VALID_VIN.lower()} "), + TEST_KEY, + "v1", + ) + + self.assertIsNotNone(normalized_row) + self.assertIsNotNone(lower_row) + assert normalized_row is not None + assert lower_row is not None + token = normalized_row[1] + bucket = normalized_row[2] + self.assertEqual(lower_row[1], token) + self.assertEqual(lower_row[2], bucket) + self.assertEqual(bucket, int(str(token)[:8], 16) % 100) + self.assertIn(bucket, range(100)) + + def test_validate_args_enforces_private_output_root(self) -> None: + private_output = ( + exporter.PRIVATE_DATA_ROOT + / "inspection_batches" + / f"unit-{uuid.uuid4().hex}.csv.gz" + ) + validated = exporter.validate_args(self._args(private_output)) + self.assertEqual(validated, private_output.resolve()) + + with tempfile.TemporaryDirectory() as temporary_directory: + external_output = Path(temporary_directory) / "private.csv.gz" + with self.assertRaises(ValueError): + exporter.validate_args(self._args(external_output)) + + allowed = exporter.validate_args( + self._args(external_output, allow_external_output=True) + ) + self.assertEqual(allowed, external_output.resolve()) + + def test_declared_output_omits_raw_vin_and_raw_obd_result(self) -> None: + self.assertNotIn("vin", exporter.OUTPUT_COLUMNS) + self.assertNotIn("raw_vin", exporter.OUTPUT_COLUMNS) + self.assertNotIn("obd_result", exporter.OUTPUT_COLUMNS) + self.assertNotIn("overall_result", exporter.OUTPUT_COLUMNS) + self.assertIn("vehicle_token", exporter.OUTPUT_COLUMNS) + self.assertIn("outcome_label_source", exporter.OUTPUT_COLUMNS) + + transformed = exporter.private_row( + self._source_row(VALID_VIN), TEST_KEY, "v1" + ) + self.assertIsNotNone(transformed) + assert transformed is not None + self.assertEqual(len(transformed), len(exporter.OUTPUT_COLUMNS)) + self.assertEqual( + transformed[exporter.OUTPUT_COLUMNS.index("outcome_label_source")], + "overall_result", + ) + self.assertNotIn(VALID_VIN, transformed) + + def test_both_exporters_share_the_versioned_label_sql(self) -> None: + history_sql = history_export.make_sql(0.5, 20260715) + + self.assertIn(exporter.OUTCOME_SELECT_SQL, exporter.EXTRACT_SQL) + self.assertIn(exporter.OUTCOME_SELECT_SQL, history_sql) + self.assertIn("s.obd_result", exporter.OUTCOME_SELECT_SQL) + self.assertIn( + "lower(btrim(s.program_type)) = 'obd'", + exporter.OUTCOME_SELECT_SQL, + ) + self.assertIn( + "upper(btrim(s.test_type)) = 'OBD'", + exporter.OUTCOME_SELECT_SQL, + ) + self.assertNotIn("WHEN 'B'", exporter.OUTCOME_SELECT_SQL) + self.assertEqual(exporter.QUERY_VERSION, "inspection_batch_v4") + self.assertEqual( + history_export.QUERY_VERSION, + "inspection_history_development_sample_v3", + ) + + def test_feasibility_sql_uses_labeled_utah_only_proxy(self) -> None: + sql_path = exporter.PROJECT_ROOT / "sql/10_episode_cohort_feasibility.sql" + sql = sql_path.read_text(encoding="utf-8") + + self.assertIn("lower(btrim(s.county)) = 'utah'", sql) + self.assertIn("lower(btrim(s.program_type)) = 'obd'", sql) + self.assertIn("upper(btrim(s.test_type)) = 'OBD'", sql) + self.assertIn("upper(btrim(s.obd_result))", sql) + self.assertIn("AS outcome_label_source", sql) + self.assertIn("'utah_obd_proxy'", sql) + self.assertNotIn("WHEN 'B' THEN", sql) + + @staticmethod + def _source_row(vin: str) -> tuple[object, ...]: + return ( + 123, + vin, + datetime(2025, 1, 15, 12, 0), + "slco", + "salt_lake", + "pass", + "overall_result", + "obd", + "INITIAL", + "EXAMPLE", + "MODEL", + 2020, + ) + + @staticmethod + def _args( + output: Path, allow_external_output: bool = False + ) -> SimpleNamespace: + start = datetime(2025, 1, 1) + return SimpleNamespace( + start=start, + end=start + timedelta(days=1), + output=output, + page_size=100, + overwrite=False, + allow_external_output=allow_external_output, + ) + + @staticmethod + def _hmac_environment( + encoded_key: str, version: str = "v1" + ) -> mock._patch_dict: + missing_key_file = Path(tempfile.gettempdir()) / ( + f"uvh-missing-key-{uuid.uuid4().hex}" + ) + return mock.patch.dict( + os.environ, + { + "VIN_HASH_KEY": encoded_key, + "VIN_HASH_KEY_FILE": str(missing_key_file), + "VIN_HASH_KEY_VERSION": version, + }, + clear=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tree_model.py b/tests/test_tree_model.py new file mode 100644 index 0000000..9eb3c44 --- /dev/null +++ b/tests/test_tree_model.py @@ -0,0 +1,261 @@ +"""Tests for the leakage-safe nonlinear candidate.""" + +from __future__ import annotations + +import importlib.util +import math +import sys +import unittest +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, List + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = PROJECT_ROOT / "scripts/train_tree_model.py" +SPEC = importlib.util.spec_from_file_location("train_tree_model", MODULE_PATH) +if SPEC is None or SPEC.loader is None: # pragma: no cover + raise RuntimeError("Could not load scripts/train_tree_model.py") +tree = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = tree +SPEC.loader.exec_module(tree) +baselines = tree.baselines + +SKLEARN_AVAILABLE = ( + importlib.util.find_spec("numpy") is not None + and importlib.util.find_spec("sklearn") is not None +) + + +def episode( + number: int, + partition: str, + year: int, + target: int, + age: float, + make: str, + county: str = "salt_lake", + audit: bool = False, +) -> baselines.EpisodeRow: + bucket = 5 if audit else 25 + (number % 70) + numeric: Dict[str, float] = { + "vehicle_age": age, + "prior_episode_count": 1.0 + number % 8, + "prior_total_attempt_count": 1.0 + number % 12, + "prior_attempt_count": 1.0 + number % 3, + "days_since_prior_episode": 300.0 + number % 150, + "days_since_prior_adverse": 200.0 + number % 500, + "prior_nonpass_rate": (number % 10) / 10.0, + } + categorical = { + "public_county": county, + "target_season": ("summer" if number % 2 else "winter"), + "prior_first_outcome": ("fail" if number % 4 == 0 else "pass"), + "prior_final_outcome": ("fail" if number % 5 == 0 else "pass"), + "last_observed_make": make, + "last_observed_model": ("model_a" if number % 3 else "model_b"), + } + return baselines.EpisodeRow( + vehicle_token="{:064x}".format(number + year * 100_000), + vehicle_bucket=bucket, + is_vin_audit=audit, + episode_number=2 + number % 10, + episode_start=datetime(year, 6, 1), + partition=partition, + target=target, + source_era="slc", + target_outcome_label_source="overall_result", + prior_first_outcome=categorical["prior_first_outcome"], + numeric=numeric, + categorical=categorical, + ) + + +def partition_rows(partition: str, year: int, count: int, offset: int) -> List[object]: + rows: List[object] = [] + for index in range(count): + number = offset + index + audit = index % 10 == 0 + age = float((number * 7) % 30) + county = "utah" if number % 2 else "salt_lake" + signal = (age > 14.0) != (county == "utah") + target = int(signal) + if number % 17 == 0: + target = 1 - target + if audit: + make = "audit_only_make" + elif partition == "train": + make = ("toyota" if number % 3 else "ford") + else: + make = ("future_make" if number % 7 == 0 else "toyota") + rows.append( + episode( + number=number, + partition=partition, + year=year, + target=target, + age=age, + make=make, + county=county, + audit=audit, + ) + ) + return rows + + +def synthetic_mart() -> baselines.MartData: + rows_by_partition = { + "train": partition_rows("train", 2022, 360, 0), + "tune": partition_rows("tune", 2023, 140, 10_000), + "calibrate": partition_rows("calibrate", 2024, 140, 20_000), + "locked_test": partition_rows("locked_test", 2025, 140, 30_000), + } + total = sum(len(rows) for rows in rows_by_partition.values()) + return baselines.MartData( + rows_by_partition=rows_by_partition, + input_rows=total, + ineligible_rows=0, + eligible_rows=total, + ) + + +class TreeEncoderTests(unittest.TestCase): + def test_future_categories_and_values_do_not_change_training_preprocessing(self) -> None: + train_rows = [ + episode(1, "train", 2022, 0, 10.0, "toyota"), + episode(2, "train", 2022, 1, 20.0, "toyota"), + episode(3, "train", 2022, 0, 30.0, "ford"), + ] + future = episode(4, "tune", 2023, 1, 9_999.0, "future_make") + encoder = tree.TreeFeatureEncoder.fit( + train_rows, min_category_count=2, max_categories=16 + ) + + self.assertEqual(encoder.numeric_medians["vehicle_age"], 20.0) + self.assertEqual( + encoder.category_code("last_observed_make", "toyota"), + tree.FIRST_KNOWN_CATEGORY_CODE, + ) + self.assertEqual( + encoder.category_code("last_observed_make", "ford"), + tree.RARE_CATEGORY_CODE, + ) + self.assertEqual( + encoder.category_code("last_observed_make", "future_make"), + tree.UNKNOWN_CATEGORY_CODE, + ) + self.assertNotIn( + "future_make", encoder.seen_categories["last_observed_make"] + ) + self.assertEqual(encoder.numeric_medians["vehicle_age"], 20.0) + self.assertTrue(set(tree.EXCLUDED_FROM_PREDICTORS).isdisjoint( + encoder.feature_names + )) + self.assertNotIn("source_era", encoder.feature_names) + self.assertNotIn("target_outcome_label_source", encoder.feature_names) + + if SKLEARN_AVAILABLE: + import numpy as np + + matrix = encoder.transform([future], np) + category_index = 2 * len(tree.NUMERIC_FEATURES) + list( + tree.CATEGORICAL_FEATURES + ).index("last_observed_make") + self.assertEqual( + matrix[0, category_index], float(tree.UNKNOWN_CATEGORY_CODE) + ) + self.assertTrue(np.isfinite(matrix).all()) + + @unittest.skipUnless(SKLEARN_AVAILABLE, "scikit-learn is not installed") + def test_convergence_gate_rejects_iteration_limit_and_nonfinite_scores(self) -> None: + import numpy as np + + at_limit = SimpleNamespace( + n_iter_=100, + train_score_=np.asarray([-1.0]), + validation_score_=np.asarray([-1.0]), + ) + converged, n_iter = tree._hist_converged(at_limit, [], 100, np) + self.assertFalse(converged) + self.assertEqual(n_iter, 100) + + nonfinite = SimpleNamespace( + n_iter_=20, + train_score_=np.asarray([-1.0, math.nan]), + validation_score_=np.asarray([-1.0]), + ) + converged, _ = tree._hist_converged(nonfinite, [], 100, np) + self.assertFalse(converged) + + +@unittest.skipUnless(SKLEARN_AVAILABLE, "scikit-learn is not installed") +class TreeTrainingTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.mart = synthetic_mart() + cls.trained = tree.train_tree_model( + mart=cls.mart, + candidates=[tree.TreeCandidate(0.1, 15, 1.0)], + min_category_count=3, + max_categories=32, + min_samples_leaf=10, + max_iter=150, + n_iter_no_change=8, + calibration_max_iter=1000, + seed=20260715, + ) + + def test_fit_excludes_audit_and_future_categories(self) -> None: + seen = self.trained.encoder.seen_categories["last_observed_make"] + self.assertNotIn("audit_only_make", seen) + self.assertNotIn("future_make", seen) + self.assertLess(self.trained.model_n_iter, self.trained.max_iter) + self.assertTrue( + all(item["eligible_for_selection"] for item in self.trained.tuning_results) + ) + + def test_locked_metrics_require_explicit_gate(self) -> None: + closed_metrics, _closed_bins = tree.evaluate_tree_model( + self.mart, + self.trained, + evaluate_locked=False, + bin_count=5, + ) + self.assertNotIn( + "locked_test", {row["partition"] for row in closed_metrics} + ) + + open_metrics, _open_bins = tree.evaluate_tree_model( + self.mart, + self.trained, + evaluate_locked=True, + bin_count=5, + ) + locked = [ + row for row in open_metrics if row["partition"] == "locked_test" + ] + self.assertTrue(locked) + self.assertIn("never_fit_audit", {row["cohort"] for row in locked}) + + def test_probabilities_are_finite(self) -> None: + probabilities = tree.tree_probabilities( + self.trained, + self.mart.rows_by_partition["calibrate"], + calibrated=True, + ) + self.assertTrue(probabilities) + self.assertTrue(all(math.isfinite(value) for value in probabilities)) + self.assertTrue(all(0.0 <= value <= 1.0 for value in probabilities)) + + def test_cli_locked_flag_defaults_closed(self) -> None: + closed = tree.parse_args(["--mart", "data/private/mart.parquet"]) + opened = tree.parse_args( + ["--mart", "data/private/mart.parquet", "--evaluate-locked"] + ) + self.assertFalse(closed.evaluate_locked) + self.assertTrue(opened.evaluate_locked) + + +if __name__ == "__main__": + unittest.main()