initial code
This commit is contained in:
parent
0952a7ffce
commit
05729fc6de
14
.env.example
Normal file
14
.env.example
Normal file
@ -0,0 +1,14 @@
|
||||
# PostgreSQL connection values. Copy names only; never commit real credentials.
|
||||
PGHOST=
|
||||
PGPORT=5432
|
||||
PGDATABASE=countydata
|
||||
PGUSER=
|
||||
PGPASSWORD=
|
||||
PGSSLMODE=require
|
||||
|
||||
# Project-specific HMAC secret for replacing VINs in private analytical data.
|
||||
# Encode exactly 32 random bytes as 64 hexadecimal characters. Alternatively,
|
||||
# leave VIN_HASH_KEY empty and store the hex key in .secrets/vin_hmac.key (0600).
|
||||
VIN_HASH_KEY=
|
||||
VIN_HASH_KEY_FILE=.secrets/vin_hmac.key
|
||||
VIN_HASH_KEY_VERSION=v1
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@ -1,6 +1,12 @@
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.ipynb_checkpoints/
|
||||
.DS_Store
|
||||
/data/
|
||||
/artifacts/
|
||||
/models/
|
||||
/.secrets/
|
||||
|
||||
5
.vscode/extensions.json
vendored
Normal file
5
.vscode/extensions.json
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"ms-ossdata.vscode-pgsql"
|
||||
]
|
||||
}
|
||||
113
.vscode/tasks.json
vendored
Normal file
113
.vscode/tasks.json
vendored
Normal file
@ -0,0 +1,113 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Utah Vehicle Health: create private VIN HMAC key",
|
||||
"type": "shell",
|
||||
"command": ".venv/bin/python scripts/create_vin_hash_key.py",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Countydata: test read-only connection",
|
||||
"type": "shell",
|
||||
"command": "set -a; source .env; set +a; .venv/bin/python scripts/check_connection.py",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: run Python tests",
|
||||
"type": "shell",
|
||||
"command": "PYTHONWARNINGS=error .venv/bin/python -m unittest discover -s tests -v",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: run aggregate feasibility check",
|
||||
"type": "shell",
|
||||
"command": "set -a; source .env; set +a; .venv/bin/python scripts/run_feasibility_check.py",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: audit source outcome mappings",
|
||||
"type": "shell",
|
||||
"command": "set -a; source .env; set +a; .venv/bin/python scripts/run_outcome_mapping_audit.py",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: export 10k development histories",
|
||||
"type": "shell",
|
||||
"command": "set -a; source .env; set +a; .venv/bin/python scripts/export_history_sample.py --vehicles 10000 --sample-percent 0.5 --overwrite",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: build private feature mart",
|
||||
"type": "shell",
|
||||
"command": ".venv/bin/python scripts/build_feature_mart.py --input data/private/development/history_sample_10000.csv.gz --memory-limit 4GB --threads 4 --overwrite",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: train baselines (2025 locked)",
|
||||
"type": "shell",
|
||||
"command": ".venv/bin/python scripts/train_baselines.py --mart data/private/marts/inspection_feature_mart.parquet --overwrite",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: train nonlinear model (2025 locked)",
|
||||
"type": "shell",
|
||||
"command": ".venv/bin/python scripts/train_tree_model.py --mart data/private/marts/inspection_feature_mart.parquet --overwrite",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: export sanitized dashboard preview",
|
||||
"type": "shell",
|
||||
"command": ".venv/bin/python scripts/export_dashboard_data.py --mart data/private/marts/inspection_feature_mart.parquet --model-manifest artifacts/private/baselines/baseline_v1/manifest.json --model-metrics artifacts/private/baselines/baseline_v1/metrics.json --model-manifest artifacts/private/tree/hist_gradient_boosting_v1/manifest.json --model-metrics artifacts/private/tree/hist_gradient_boosting_v1/metrics.json --development-preview --overwrite",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: run dashboard contract tests",
|
||||
"type": "shell",
|
||||
"command": "node --test dashboard/tests/contract.test.mjs",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Utah Vehicle Health: run all tests",
|
||||
"dependsOrder": "sequence",
|
||||
"dependsOn": [
|
||||
"Utah Vehicle Health: run Python tests",
|
||||
"Utah Vehicle Health: run dashboard contract tests"
|
||||
],
|
||||
"problemMatcher": []
|
||||
}
|
||||
]
|
||||
}
|
||||
126
README.md
126
README.md
@ -1,3 +1,127 @@
|
||||
# SummerProject2026
|
||||
|
||||
Kevin Bell's summer 2026 project
|
||||
Kevin Bell's summer 2026 data project using Utah county vehicle-registration
|
||||
and emissions-inspection data.
|
||||
|
||||
See the [data inventory](docs/data_inventory.md) and the data-backed
|
||||
[project shortlist](docs/project_options.md).
|
||||
|
||||
## Selected project
|
||||
|
||||
The selected direction is **Utah Vehicle Health**, an explainable model and
|
||||
dashboard for the first-attempt outcome of a vehicle's next inspection episode.
|
||||
Start with the [project charter](docs/project_charter.md), then use the
|
||||
[modeling protocol](docs/modeling_protocol.md) and
|
||||
[dashboard specification](docs/dashboard_spec.md) as the project contracts.
|
||||
The latest private-sample pipeline findings are summarized in
|
||||
[development results](docs/development_results.md), with intended use and
|
||||
limitations consolidated in the [model card](docs/model_card.md).
|
||||
|
||||
## Safe database access in VS Code
|
||||
|
||||
The local `.env` file contains the standard PostgreSQL connection variables and
|
||||
is intentionally excluded from Git. Never put those values in source code,
|
||||
screenshots, browser JavaScript, or a Bolt project.
|
||||
|
||||
1. Install the workspace-recommended **PostgreSQL** extension by Microsoft
|
||||
(`ms-ossdata.vscode-pgsql`).
|
||||
2. In VS Code, run **Tasks: Run Task** and choose
|
||||
**Countydata: test read-only connection**. It loads `.env`, requires TLS, and
|
||||
forces the PostgreSQL session into read-only mode.
|
||||
3. To browse the server visually, open the PostgreSQL sidebar and add a
|
||||
connection using the values in `.env`. Set SSL mode to **Require**, save the
|
||||
profile at **User** scope, and store the password in macOS Keychain rather
|
||||
than workspace settings.
|
||||
4. Connect to `countydata` and run [sql/00_read_only_connection_check.sql](sql/00_read_only_connection_check.sql).
|
||||
Safe aggregate examples are in [sql/01_safe_data_overview.sql](sql/01_safe_data_overview.sql).
|
||||
|
||||
The selected project's first aggregate cohort check is
|
||||
[sql/10_episode_cohort_feasibility.sql](sql/10_episode_cohort_feasibility.sql).
|
||||
Private bounded extraction is handled by
|
||||
[scripts/export_inspection_batch.py](scripts/export_inspection_batch.py); it
|
||||
requires the project-specific `VIN_HASH_KEY` described in `.env.example` and
|
||||
never writes a raw VIN.
|
||||
|
||||
The current database login has write privileges even though this project only
|
||||
needs reads. Ask the database administrator for a dedicated read-only role
|
||||
before connecting any deployed service. Until then, always use explicit
|
||||
read-only transactions.
|
||||
|
||||
## Reproducible development workflow
|
||||
|
||||
Create the local environment once:
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install -r requirements.txt
|
||||
.venv/bin/python scripts/create_vin_hash_key.py
|
||||
```
|
||||
|
||||
The recommended way to run the pipeline is **Tasks: Run Task** in VS Code. The
|
||||
tasks preserve the intended order:
|
||||
|
||||
1. Test the read-only TLS connection.
|
||||
2. Run the source outcome-mapping and aggregate feasibility audits.
|
||||
3. Export the private 10,000-vehicle development histories.
|
||||
4. Build the private DuckDB warehouse and Parquet feature mart.
|
||||
5. Run every test.
|
||||
6. Train the baselines while leaving the 2025 test partition locked.
|
||||
7. Train the nonlinear comparison while leaving the same partition locked.
|
||||
|
||||
The development history sample is deliberately not population-representative.
|
||||
It exists to exercise feature engineering and modeling before a complete,
|
||||
contiguous bounded extraction is approved. Generated extracts, marts, database
|
||||
files, keys, and model artifacts remain under Git-ignored private paths.
|
||||
|
||||
The 2025 metrics are not calculated by the normal training task. Unlocking them
|
||||
requires the conspicuous `--evaluate-locked` flag after the feature set,
|
||||
hyperparameters, calibration method, and reporting plan are frozen.
|
||||
That gate was opened once on the page-sampled development extract during live
|
||||
verification; the normal artifact paths were then regenerated closed. The
|
||||
audit trail is in [development results](docs/development_results.md), and no
|
||||
further 2025-informed tuning is permitted.
|
||||
|
||||
## Sanitized dashboard preview
|
||||
|
||||
After training both closed model bundles, run **Tasks: Run Task** and choose
|
||||
**Utah Vehicle Health: export sanitized dashboard preview**. The exporter
|
||||
publishes only suppression-reviewed JSON under `dashboard/public/data`; it
|
||||
refuses the sampled mart unless the explicit development-preview flag is used
|
||||
and refuses model artifacts containing holdout metrics.
|
||||
|
||||
Preview the site without exposing the repository root:
|
||||
|
||||
```bash
|
||||
node dashboard/server.mjs
|
||||
```
|
||||
|
||||
Then open `http://localhost:4173`. Do not serve the repository root, because it
|
||||
contains the local connection profile and private ignored directories.
|
||||
|
||||
The checked-in bundle is visibly labeled as a non-population development
|
||||
preview. Before a real public-data release, rerun the frozen pipeline on an
|
||||
approved complete extraction and pass the publication review documented in
|
||||
[the dashboard specification](docs/dashboard_spec.md).
|
||||
|
||||
Local and Bolt handoff instructions are in
|
||||
[dashboard/README.md](dashboard/README.md). Bolt can import a GitHub repository,
|
||||
but the hosted site root must be the `dashboard` directory; never copy `.env`,
|
||||
private data, model artifacts, or database tooling into a web project.
|
||||
|
||||
## Data safety and publishing
|
||||
|
||||
The source contains direct identifiers and operational data, including VINs,
|
||||
plates, user records, sessions, and upload metadata. A public dashboard should
|
||||
contain only de-identified aggregates and model outputs with minimum group-size
|
||||
suppression.
|
||||
|
||||
Recommended deployment flow:
|
||||
|
||||
```text
|
||||
countydata (read-only) -> local ETL/modeling -> sanitized aggregate tables/files
|
||||
-> Bolt-hosted dashboard
|
||||
```
|
||||
|
||||
Do not connect a browser directly to `countydata`. If the dashboard must refresh
|
||||
automatically, use a scheduled backend job with a dedicated read-only database
|
||||
role and copy only approved aggregate results into the hosted application.
|
||||
|
||||
67
dashboard/README.md
Normal file
67
dashboard/README.md
Normal file
@ -0,0 +1,67 @@
|
||||
# Utah Vehicle Health dashboard
|
||||
|
||||
This directory is a dependency-free static prototype. It uses semantic HTML,
|
||||
CSS, vanilla ES modules, and inline SVG generated from approved aggregate JSON.
|
||||
It has no database client, server credential, analytics SDK, or external CDN.
|
||||
|
||||
## Run locally from the repository root
|
||||
|
||||
```bash
|
||||
node dashboard/server.mjs
|
||||
```
|
||||
|
||||
Open `http://127.0.0.1:4173`. Do not open `index.html` directly with a `file:`
|
||||
URL; browsers will block the JSON module requests. To use another port:
|
||||
|
||||
```bash
|
||||
PORT=8080 node dashboard/server.mjs
|
||||
```
|
||||
|
||||
Run the dependency-free contract checks with:
|
||||
|
||||
```bash
|
||||
node --test dashboard/tests/contract.test.mjs
|
||||
```
|
||||
|
||||
## Public-data boundary
|
||||
|
||||
The browser requires and validates these files beneath `public/data/`:
|
||||
|
||||
- `data_manifest.json`
|
||||
- `overview_period_county.json`
|
||||
- `age_risk_curve.json`
|
||||
- `cohort_scorecard.json`
|
||||
- `model_diagnostics.json`
|
||||
- `coverage_quality.json`
|
||||
- `filter_catalog.json`
|
||||
- `sha256_manifest.json`
|
||||
|
||||
If any file is missing, malformed, has inconsistent publication flags, or
|
||||
contains a denied identifier-like field, the dashboard shows an unavailable
|
||||
state and no estimates. Regenerate assets with the repository's private local
|
||||
pipeline; never hand-edit public JSON to bypass suppression.
|
||||
|
||||
The next-test estimator is intentionally disabled. It must remain disabled
|
||||
until a separately reviewed and suppressed `prediction_lookup` contract is
|
||||
approved. Do not add a connection from this site to `countydata`.
|
||||
|
||||
## Import into Bolt
|
||||
|
||||
The recommended publication boundary is a separate static Bolt project (and,
|
||||
ideally, a separate deployment repository) containing only the contents of
|
||||
`dashboard/`. Upload or copy this directory's contents so `index.html` is at the
|
||||
new project root. There is no install or build step. If Bolt asks for a preview
|
||||
command, use `node server.mjs` with `HOST=0.0.0.0`; Bolt supplies `PORT`.
|
||||
|
||||
Before publishing, confirm that `/public/data/data_manifest.json` resolves,
|
||||
the development-preview banner remains visible, and the contract test passes.
|
||||
The generated JSON bundle must also complete its privacy review.
|
||||
|
||||
Importing the whole source repository is a discouraged fallback because Bolt
|
||||
would receive SQL, database tooling, and other files that are not needed by the
|
||||
public site. If that has already happened, set the project/working root to
|
||||
`dashboard`, never serve the repository root, and create a dashboard-only
|
||||
project before production publication. Do not copy private `data/`,
|
||||
`artifacts/`, `models/`, `.env`, or database tooling into the public project.
|
||||
|
||||
No publishing action is performed by this repository.
|
||||
379
dashboard/index.html
Normal file
379
dashboard/index.html
Normal file
@ -0,0 +1,379 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="Utah Vehicle Health — an accessible, aggregate view of emissions-inspection outcomes.">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<title>Overview · Utah Vehicle Health</title>
|
||||
<link rel="stylesheet" href="./styles.css">
|
||||
<script type="module" src="./js/app.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<a class="skip-link" href="#main-content">Skip to dashboard content</a>
|
||||
|
||||
<div id="preview-banner" class="preview-banner" role="status" hidden>
|
||||
<span class="preview-banner__icon" aria-hidden="true">◆</span>
|
||||
<div>
|
||||
<strong id="preview-title">Development preview</strong>
|
||||
<span id="preview-copy">Sample results are not population estimates and must not be used for individual decisions.</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<header class="site-header">
|
||||
<div class="site-header__inner">
|
||||
<a class="brand" href="#overview" aria-label="Utah Vehicle Health overview">
|
||||
<svg class="brand__mark" viewBox="0 0 56 56" aria-hidden="true">
|
||||
<path d="M9 43 23 17l7 12 6-9 12 23H9Z" fill="currentColor"></path>
|
||||
<path d="m18 43 9-16 6 10 4-6 6 12H18Z" fill="var(--sand-100)"></path>
|
||||
</svg>
|
||||
<span>
|
||||
<span class="brand__name">Utah Vehicle Health</span>
|
||||
<span class="brand__tagline">Next-episode inspection insights</span>
|
||||
</span>
|
||||
</a>
|
||||
|
||||
<div class="header-status" id="header-status" aria-live="polite">
|
||||
<span class="status-dot" aria-hidden="true"></span>
|
||||
<span>Loading approved aggregates…</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav class="primary-nav" aria-label="Dashboard views">
|
||||
<div class="primary-nav__inner">
|
||||
<a href="#overview" data-route="overview" aria-current="page">Overview</a>
|
||||
<a href="#reliability" data-route="reliability">Reliability explorer</a>
|
||||
<a href="#estimator" data-route="estimator">Next-test estimator</a>
|
||||
<a href="#methods" data-route="methods">Data & methods</a>
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main id="main-content" tabindex="-1">
|
||||
<section class="view" id="view-overview" data-view="overview" aria-labelledby="overview-heading">
|
||||
<div class="page-shell">
|
||||
<header class="page-heading page-heading--split">
|
||||
<div>
|
||||
<p class="eyebrow">Inspection outcomes over time</p>
|
||||
<h1 id="overview-heading">A clearer view of the next inspection</h1>
|
||||
<p class="lede">Explore aggregate first-attempt outcomes for returning vehicles in participating Utah county feeds. Non-pass combines fail, reject, and abort.</p>
|
||||
</div>
|
||||
<div class="data-stamp" aria-label="Dataset status">
|
||||
<span>Data through</span>
|
||||
<strong id="data-cutoff">Unavailable</strong>
|
||||
<small id="model-version">Waiting for manifest</small>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="unavailable-state" data-unavailable hidden role="status">
|
||||
<div class="unavailable-state__icon" aria-hidden="true">!</div>
|
||||
<div>
|
||||
<h2>Dashboard data is unavailable</h2>
|
||||
<p data-unavailable-message>The approved aggregate files could not be validated. No estimates are being shown.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section class="kpi-grid" aria-label="Overview statistics" data-requires-data>
|
||||
<article class="kpi-card">
|
||||
<span class="kpi-card__label">Published support</span>
|
||||
<strong id="kpi-eligible">—</strong>
|
||||
<small>Rounded total; suppressed cells omitted</small>
|
||||
</article>
|
||||
<article class="kpi-card kpi-card--pass">
|
||||
<span class="kpi-card__label">Pass rate</span>
|
||||
<strong id="kpi-pass">—</strong>
|
||||
<small>Recognized first-attempt outcomes</small>
|
||||
</article>
|
||||
<article class="kpi-card kpi-card--nonpass">
|
||||
<span class="kpi-card__label">Non-pass rate</span>
|
||||
<strong id="kpi-nonpass">—</strong>
|
||||
<small>Fail + reject + abort</small>
|
||||
</article>
|
||||
<article class="kpi-card">
|
||||
<span class="kpi-card__label">Covered counties</span>
|
||||
<strong id="kpi-counties">—</strong>
|
||||
<small>Not statewide coverage</small>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<div class="dashboard-grid dashboard-grid--wide" data-requires-data>
|
||||
<article class="panel panel--wide">
|
||||
<header class="panel__header">
|
||||
<div>
|
||||
<p class="eyebrow">Quarterly outcomes</p>
|
||||
<h2>First-attempt non-pass trend</h2>
|
||||
</div>
|
||||
<div class="chart-legend" aria-label="Outcome legend">
|
||||
<span><i class="legend-swatch legend-swatch--fail"></i>Non-pass (fail + reject + abort)</span>
|
||||
</div>
|
||||
</header>
|
||||
<div id="outcome-trend-chart" class="chart chart--large" role="img" aria-label="Quarterly inspection outcome rates"></div>
|
||||
<p class="chart-note" id="outcome-trend-note">Partial periods and source changes are called out when present.</p>
|
||||
</article>
|
||||
|
||||
<article class="panel">
|
||||
<header class="panel__header">
|
||||
<div>
|
||||
<p class="eyebrow">Feed availability</p>
|
||||
<h2>County coverage</h2>
|
||||
</div>
|
||||
</header>
|
||||
<div id="utah-coverage-map" class="utah-map" role="img" aria-label="Utah county-feed coverage"></div>
|
||||
<div id="county-coverage-list" class="county-list" aria-label="County availability list"></div>
|
||||
<p class="chart-note">Teal counties appear in the approved inspection feeds. Gray counties are unavailable, not zero.</p>
|
||||
</article>
|
||||
|
||||
<article class="panel">
|
||||
<header class="panel__header">
|
||||
<div>
|
||||
<p class="eyebrow">Age pattern</p>
|
||||
<h2>Non-pass risk by vehicle age</h2>
|
||||
</div>
|
||||
</header>
|
||||
<div id="age-risk-chart" class="chart" role="img" aria-label="Published non-pass risk by vehicle-age band"></div>
|
||||
<p class="chart-note">The development bundle does not include confidence intervals; published aggregate rates are shown as points.</p>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<aside class="callout" data-requires-data>
|
||||
<span class="callout__icon" aria-hidden="true">i</span>
|
||||
<p><strong>What this measures:</strong> emissions-inspection outcomes for covered feeds. It is not a diagnosis of mechanical condition, safety, roadworthiness, or legal compliance.</p>
|
||||
</aside>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-reliability" data-view="reliability" aria-labelledby="reliability-heading" hidden>
|
||||
<div class="page-shell">
|
||||
<header class="page-heading">
|
||||
<p class="eyebrow">Supported aggregate cohorts</p>
|
||||
<h1 id="reliability-heading">Reliability explorer</h1>
|
||||
<p class="lede">Compare published observed next-episode non-pass rates. Every result is a supported aggregate cohort—not an individual prediction.</p>
|
||||
</header>
|
||||
|
||||
<div class="unavailable-state" data-unavailable hidden role="status">
|
||||
<div class="unavailable-state__icon" aria-hidden="true">!</div>
|
||||
<div><h2>Explorer unavailable</h2><p data-unavailable-message>Approved scorecards could not be validated.</p></div>
|
||||
</div>
|
||||
|
||||
<div class="explorer-layout" data-requires-data>
|
||||
<aside class="filter-panel" aria-labelledby="filter-heading">
|
||||
<div class="filter-panel__heading">
|
||||
<h2 id="filter-heading">Refine cohorts</h2>
|
||||
<button class="text-button" id="reset-filters" type="button">Reset</button>
|
||||
</div>
|
||||
<form id="explorer-filters">
|
||||
<label class="field">
|
||||
<span>Make or model</span>
|
||||
<input id="cohort-search" type="search" autocomplete="off" placeholder="Search supported cohorts">
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>County</span>
|
||||
<select id="filter-county"><option value="">All covered counties</option></select>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Vehicle age</span>
|
||||
<select id="filter-age"><option value="">All age bands</option></select>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Fuel</span>
|
||||
<select id="filter-fuel"><option value="">All supported fuels</option></select>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Program</span>
|
||||
<select id="filter-program"><option value="">All approved programs</option></select>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Period</span>
|
||||
<select id="filter-period"><option value="">All published periods</option></select>
|
||||
</label>
|
||||
<fieldset class="segmented-control">
|
||||
<legend>Risk view</legend>
|
||||
<label><input type="radio" name="risk-view" value="observed" checked><span>Observed</span></label>
|
||||
<label><input type="radio" name="risk-view" value="adjusted"><span>Model-adjusted</span></label>
|
||||
</fieldset>
|
||||
</form>
|
||||
</aside>
|
||||
|
||||
<div class="explorer-results">
|
||||
<div class="result-toolbar">
|
||||
<p id="result-summary" aria-live="polite">Loading supported cohorts…</p>
|
||||
<label class="field field--inline">
|
||||
<span>Sort</span>
|
||||
<select id="result-sort">
|
||||
<option value="support">Largest support</option>
|
||||
<option value="risk-desc">Highest non-pass risk</option>
|
||||
<option value="risk-asc">Lowest non-pass risk</option>
|
||||
<option value="name">Make and model</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<article class="panel">
|
||||
<header class="panel__header">
|
||||
<div><p class="eyebrow">Published aggregate comparison</p><h2>Supported cohort scorecard</h2></div>
|
||||
</header>
|
||||
<div id="cohort-dot-plot" class="chart chart--scorecard" role="img" aria-label="Ranked published cohort non-pass rates"></div>
|
||||
</article>
|
||||
|
||||
<div id="cohort-cards" class="cohort-cards"></div>
|
||||
<div class="empty-results" id="empty-results" hidden>
|
||||
<h2>No supported cohorts match</h2>
|
||||
<p>Try clearing one or more filters. Suppressed cohorts are never included in the browser bundle.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-estimator" data-view="estimator" aria-labelledby="estimator-heading" hidden>
|
||||
<div class="page-shell page-shell--narrow">
|
||||
<header class="page-heading">
|
||||
<p class="eyebrow">Cohort estimate · not an individual diagnosis</p>
|
||||
<h1 id="estimator-heading">Next-test risk estimator</h1>
|
||||
<p class="lede">This planned tool will combine approved, coarsened attributes to return a calibrated next-episode non-pass probability.</p>
|
||||
</header>
|
||||
|
||||
<div class="locked-panel" role="status" aria-labelledby="estimator-status-title">
|
||||
<span class="locked-panel__icon" aria-hidden="true">◇</span>
|
||||
<div>
|
||||
<p class="eyebrow">Intentionally disabled</p>
|
||||
<h2 id="estimator-status-title">An approved prediction lookup is not available</h2>
|
||||
<p>The estimator will remain off until a privacy-reviewed, suppressed <code>prediction_lookup</code> is published. The current model diagnostics are not enough to serve individual or row-level estimates.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form class="estimator-form" aria-describedby="estimator-disabled-copy">
|
||||
<fieldset disabled>
|
||||
<legend>Coarsened cohort attributes</legend>
|
||||
<div class="form-grid">
|
||||
<label class="field"><span>County</span><select><option>Select a covered county</option></select></label>
|
||||
<label class="field"><span>Supported make and model</span><select><option>Select a cohort</option></select></label>
|
||||
<label class="field"><span>Vehicle-age band</span><select><option>Select an age band</option></select></label>
|
||||
<label class="field"><span>Fuel</span><select><option>Select a supported fuel</option></select></label>
|
||||
<label class="field"><span>Prior episode outcome</span><select><option>Pass / fail / reject / abort</option></select></label>
|
||||
<label class="field"><span>Time since prior episode</span><select><option>Select a coarsened interval</option></select></label>
|
||||
<label class="field"><span>Season</span><select><option>Select a season</option></select></label>
|
||||
<label class="field"><span>Program category</span><select><option>Select an approved program</option></select></label>
|
||||
</div>
|
||||
<button class="button" type="button">Estimate cohort risk</button>
|
||||
</fieldset>
|
||||
</form>
|
||||
<p id="estimator-disabled-copy" class="form-note">No VIN, plate, exact address, station, free text, or current-test diagnostic will ever be requested.</p>
|
||||
|
||||
<section class="method-preview" aria-labelledby="future-output-heading">
|
||||
<div>
|
||||
<p class="eyebrow">Future approved output</p>
|
||||
<h2 id="future-output-heading">Designed for calibrated context</h2>
|
||||
</div>
|
||||
<ul class="check-list">
|
||||
<li>Calibrated non-pass probability and uncertainty</li>
|
||||
<li>Relevant aggregate baseline</li>
|
||||
<li>High-level factor contributions</li>
|
||||
<li>Support and coverage limitations</li>
|
||||
</ul>
|
||||
</section>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-methods" data-view="methods" aria-labelledby="methods-heading" hidden>
|
||||
<div class="page-shell">
|
||||
<header class="page-heading">
|
||||
<p class="eyebrow">Transparent by design</p>
|
||||
<h1 id="methods-heading">Data & methods</h1>
|
||||
<p class="lede">How episodes, labels, temporal evaluation, coverage limits, and privacy controls shape every published result.</p>
|
||||
</header>
|
||||
|
||||
<div class="unavailable-state" data-unavailable hidden role="status">
|
||||
<div class="unavailable-state__icon" aria-hidden="true">!</div>
|
||||
<div><h2>Live diagnostics unavailable</h2><p data-unavailable-message>Static methodology remains below; no performance values are being shown.</p></div>
|
||||
</div>
|
||||
|
||||
<section class="method-grid" aria-label="Core definitions">
|
||||
<article class="method-card">
|
||||
<span class="method-card__number">01</span>
|
||||
<h2>Prediction unit</h2>
|
||||
<p>The first attempt of the next inspection episode. Attempts no more than 30 days apart form one episode.</p>
|
||||
</article>
|
||||
<article class="method-card">
|
||||
<span class="method-card__number">02</span>
|
||||
<h2>Binary target</h2>
|
||||
<p>Pass versus non-pass. Non-pass combines fail, reject, and abort; blanks and unrecognized outcomes are not labels.</p>
|
||||
</article>
|
||||
<article class="method-card">
|
||||
<span class="method-card__number">03</span>
|
||||
<h2>Point-in-time features</h2>
|
||||
<p>Every history window ends before the target episode. Current-test results, diagnostics, stations, and later attempts are excluded.</p>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="panel" data-requires-data>
|
||||
<header class="panel__header"><div><p class="eyebrow">Availability by source era</p><h2>Coverage timeline</h2></div></header>
|
||||
<div id="coverage-heatmap" class="heatmap" role="img" aria-label="Inspection source availability by year"></div>
|
||||
<p class="chart-note">The <code>slc</code> and <code>slco</code> labels are source eras for Salt Lake County, not separate physical counties.</p>
|
||||
</section>
|
||||
|
||||
<section class="split-section" aria-labelledby="split-heading">
|
||||
<div class="split-section__copy">
|
||||
<p class="eyebrow">No random row split</p>
|
||||
<h2 id="split-heading">Chronological evaluation</h2>
|
||||
<p>Historical events establish prior context. The 2025 development-sample holdout was evaluated once after model choices were frozen, so it is no longer a pristine final test. A later complete-data run is confirmatory; the partial 2026 feed is monitoring only.</p>
|
||||
</div>
|
||||
<ol class="timeline">
|
||||
<li><span>2010–15</span><strong>Context</strong><small>History only</small></li>
|
||||
<li><span>2016–22</span><strong>Train</strong><small>Fit preprocessing + model</small></li>
|
||||
<li><span>2023</span><strong>Tune</strong><small>Select regularization</small></li>
|
||||
<li><span>2024</span><strong>Calibrate</strong><small>Scale probabilities</small></li>
|
||||
<li class="timeline__holdout"><span>2025</span><strong>One-time holdout</strong><small>Already evaluated</small></li>
|
||||
<li><span>2026</span><strong>Shadow</strong><small>Partial-period drift</small></li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
<div class="dashboard-grid" data-requires-data>
|
||||
<article class="panel">
|
||||
<header class="panel__header"><div><p class="eyebrow" id="diagnostic-scope-label">Model diagnostics</p><h2>Published model diagnostics</h2></div></header>
|
||||
<div class="diagnostic-kpis">
|
||||
<div><span>PR-AUC</span><strong id="diagnostic-ap">—</strong></div>
|
||||
<div><span>Brier score</span><strong id="diagnostic-brier">—</strong></div>
|
||||
<div><span>Support</span><strong id="diagnostic-n">—</strong></div>
|
||||
</div>
|
||||
<div id="calibration-chart" class="chart" role="img" aria-label="Predicted versus observed non-pass risk calibration"></div>
|
||||
</article>
|
||||
|
||||
<article class="panel">
|
||||
<header class="panel__header"><div><p class="eyebrow">Interpret with care</p><h2>Known limitations</h2></div></header>
|
||||
<ul class="limitation-list" id="manifest-limitations">
|
||||
<li>Participating inspection feeds do not cover all 29 counties.</li>
|
||||
<li>Feed and program changes can resemble real-world trends.</li>
|
||||
<li>DMV history has a 2021 gap and ends in March 2024.</li>
|
||||
<li>Reject and abort can reflect process or readiness issues.</li>
|
||||
<li>Association is not causation or a mechanical diagnosis.</li>
|
||||
</ul>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<section class="privacy-panel">
|
||||
<div><p class="eyebrow">Publication boundary</p><h2>Only approved aggregates reach this site</h2></div>
|
||||
<div class="privacy-flow" aria-label="Private-to-public data flow">
|
||||
<span class="privacy-flow__private">Private source<br><small>Read-only local pipeline</small></span>
|
||||
<span aria-hidden="true">→</span>
|
||||
<span>Suppression<br><small>Minimum support + complements</small></span>
|
||||
<span aria-hidden="true">→</span>
|
||||
<span class="privacy-flow__public">Public assets<br><small>Aggregate JSON only</small></span>
|
||||
</div>
|
||||
<p>Direct identifiers, private vehicle tokens, plates, ZIPs, stations, raw records, and row-level predictions are outside the public data contract.</p>
|
||||
</section>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer class="site-footer">
|
||||
<div class="page-shell site-footer__inner">
|
||||
<div><strong>Utah Vehicle Health</strong><p>Aggregate emissions-inspection research.</p></div>
|
||||
<p>Not a diagnosis, certification, safety assessment, or guarantee of an inspection outcome.</p>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
<div id="route-announcer" class="sr-only" aria-live="polite"></div>
|
||||
<noscript><div class="noscript">JavaScript is required to validate and display the approved aggregate data files. No estimates are shown without validation.</div></noscript>
|
||||
</body>
|
||||
</html>
|
||||
352
dashboard/js/app.js
Normal file
352
dashboard/js/app.js
Normal file
@ -0,0 +1,352 @@
|
||||
import { loadDashboardData } from "./data.js";
|
||||
import {
|
||||
aggregateOverview,
|
||||
compactNumber,
|
||||
percent,
|
||||
renderAgeRisk,
|
||||
renderCohortDotPlot,
|
||||
renderCoverageHeatmap,
|
||||
renderNoCalibration,
|
||||
renderOutcomeTrend,
|
||||
renderUtahCoverage,
|
||||
} from "./charts.js";
|
||||
|
||||
const ROUTES = Object.freeze({
|
||||
overview: "Overview",
|
||||
reliability: "Reliability explorer",
|
||||
estimator: "Next-test estimator",
|
||||
methods: "Data & methods",
|
||||
});
|
||||
|
||||
const UTAH_COUNTIES = [
|
||||
"Beaver",
|
||||
"Box Elder",
|
||||
"Cache",
|
||||
"Carbon",
|
||||
"Daggett",
|
||||
"Davis",
|
||||
"Duchesne",
|
||||
"Emery",
|
||||
"Garfield",
|
||||
"Grand",
|
||||
"Iron",
|
||||
"Juab",
|
||||
"Kane",
|
||||
"Millard",
|
||||
"Morgan",
|
||||
"Piute",
|
||||
"Rich",
|
||||
"Salt Lake",
|
||||
"San Juan",
|
||||
"Sanpete",
|
||||
"Sevier",
|
||||
"Summit",
|
||||
"Tooele",
|
||||
"Uintah",
|
||||
"Utah",
|
||||
"Wasatch",
|
||||
"Washington",
|
||||
"Wayne",
|
||||
"Weber",
|
||||
];
|
||||
|
||||
let dashboardData = null;
|
||||
let visibleScorecards = [];
|
||||
|
||||
function byId(id) {
|
||||
return document.getElementById(id);
|
||||
}
|
||||
|
||||
function setText(id, value) {
|
||||
const element = byId(id);
|
||||
if (element) element.textContent = value;
|
||||
}
|
||||
|
||||
function normalizeCounty(value) {
|
||||
return String(value).trim().toLowerCase().replaceAll("_", " ").replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function displayCategory(value) {
|
||||
return String(value)
|
||||
.replaceAll("_", " ")
|
||||
.toLowerCase()
|
||||
.replace(/\b\w/g, (letter) => letter.toUpperCase());
|
||||
}
|
||||
|
||||
function routeFromHash() {
|
||||
const candidate = window.location.hash.replace(/^#/, "").toLowerCase();
|
||||
return Object.hasOwn(ROUTES, candidate) ? candidate : "overview";
|
||||
}
|
||||
|
||||
function showRoute({ announce = true, focus = false } = {}) {
|
||||
const route = routeFromHash();
|
||||
for (const view of document.querySelectorAll("[data-view]")) {
|
||||
view.hidden = view.dataset.view !== route;
|
||||
}
|
||||
for (const link of document.querySelectorAll("[data-route]")) {
|
||||
if (link.dataset.route === route) link.setAttribute("aria-current", "page");
|
||||
else link.removeAttribute("aria-current");
|
||||
}
|
||||
document.title = `${ROUTES[route]} · Utah Vehicle Health`;
|
||||
if (announce) setText("route-announcer", `${ROUTES[route]} view`);
|
||||
if (focus) {
|
||||
const heading = document.querySelector(`[data-view="${route}"] h1`);
|
||||
if (heading) {
|
||||
heading.setAttribute("tabindex", "-1");
|
||||
heading.focus({ preventScroll: true });
|
||||
heading.scrollIntoView({ behavior: "smooth", block: "start" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function initializeRouting() {
|
||||
showRoute({ announce: false, focus: false });
|
||||
window.addEventListener("hashchange", () => showRoute({ focus: true }));
|
||||
}
|
||||
|
||||
function setHeaderStatus(state, message) {
|
||||
const status = byId("header-status");
|
||||
if (!status) return;
|
||||
status.dataset.state = state;
|
||||
const text = status.querySelector("span:last-child");
|
||||
if (text) text.textContent = message;
|
||||
}
|
||||
|
||||
function setPreviewBanner(manifest) {
|
||||
const preview = manifest.development_preview || !manifest.population_estimate_allowed;
|
||||
const banner = byId("preview-banner");
|
||||
banner.hidden = !preview;
|
||||
if (!preview) return;
|
||||
setText("preview-title", "Development preview");
|
||||
setText(
|
||||
"preview-copy",
|
||||
"These suppressed sample aggregates are not population estimates and must not be used for individual decisions.",
|
||||
);
|
||||
}
|
||||
|
||||
function setUnavailable(error) {
|
||||
const safeMessage =
|
||||
error instanceof Error && error.message
|
||||
? `${error.message} No estimates are being shown.`
|
||||
: "The approved aggregate files could not be validated. No estimates are being shown.";
|
||||
for (const state of document.querySelectorAll("[data-unavailable]")) {
|
||||
state.hidden = false;
|
||||
const copy = state.querySelector("[data-unavailable-message]");
|
||||
if (copy) copy.textContent = safeMessage;
|
||||
}
|
||||
for (const region of document.querySelectorAll("[data-requires-data]")) {
|
||||
region.hidden = true;
|
||||
}
|
||||
const banner = byId("preview-banner");
|
||||
banner.hidden = false;
|
||||
setText("preview-title", "Data unavailable");
|
||||
setText("preview-copy", "This static shell could not validate every required public aggregate. No fallback or estimated values are displayed.");
|
||||
setHeaderStatus("error", "Approved data unavailable");
|
||||
}
|
||||
|
||||
function populateSelect(id, values, formatter = displayCategory) {
|
||||
const select = byId(id);
|
||||
if (!select) return;
|
||||
for (const value of values) {
|
||||
const option = document.createElement("option");
|
||||
option.value = String(value);
|
||||
option.textContent = formatter(value);
|
||||
select.append(option);
|
||||
}
|
||||
}
|
||||
|
||||
function disableUnsupportedFilter(id, explanation) {
|
||||
const select = byId(id);
|
||||
if (!select) return;
|
||||
select.disabled = true;
|
||||
select.title = explanation;
|
||||
select.options[0].textContent = explanation;
|
||||
}
|
||||
|
||||
function renderCountyList(coveredCounties) {
|
||||
const container = byId("county-coverage-list");
|
||||
container.replaceChildren();
|
||||
const normalized = new Set(coveredCounties.map(normalizeCounty));
|
||||
for (const county of UTAH_COUNTIES) {
|
||||
const chip = document.createElement("span");
|
||||
chip.className = "county-chip";
|
||||
chip.dataset.covered = String(normalized.has(normalizeCounty(county)));
|
||||
chip.textContent = county;
|
||||
container.append(chip);
|
||||
}
|
||||
}
|
||||
|
||||
function renderOverview(data) {
|
||||
const rows = data.overview.rows;
|
||||
const periods = aggregateOverview(rows);
|
||||
const totalSupport = rows.reduce((sum, row) => sum + row.support_rounded, 0);
|
||||
const weightedNonpass = rows.reduce(
|
||||
(sum, row) => sum + row.support_rounded * row.nonpass_rate,
|
||||
0,
|
||||
);
|
||||
const nonpassRate = totalSupport > 0 ? weightedNonpass / totalSupport : 0;
|
||||
const counties = [...new Set(rows.map((row) => row.public_county))].sort();
|
||||
const latest = periods[periods.length - 1];
|
||||
|
||||
setText("kpi-eligible", `≈${compactNumber(totalSupport)}`);
|
||||
setText("kpi-pass", `≈${percent(1 - nonpassRate)}`);
|
||||
setText("kpi-nonpass", `≈${percent(nonpassRate)}`);
|
||||
setText("kpi-counties", String(counties.length));
|
||||
setText("data-cutoff", latest ? `${latest.year} Q${latest.quarter}` : "Unavailable");
|
||||
setText(
|
||||
"model-version",
|
||||
`Release ${data.manifest.release_id.slice(0, 8)} · ${data.manifest.model_versions.join(" / ")}`,
|
||||
);
|
||||
renderOutcomeTrend(byId("outcome-trend-chart"), rows);
|
||||
setText(
|
||||
"outcome-trend-note",
|
||||
"The current approved bundle publishes binary non-pass rates only. Four-class outcome mix and blank rates are not inferred or displayed.",
|
||||
);
|
||||
renderAgeRisk(byId("age-risk-chart"), data.ageRisk.rows);
|
||||
renderUtahCoverage(byId("utah-coverage-map"), counties);
|
||||
renderCountyList(counties);
|
||||
}
|
||||
|
||||
function scorecardLabel(row) {
|
||||
return `${row.prior_make} ${row.prior_model}`.trim();
|
||||
}
|
||||
|
||||
function sortedScorecards(rows, mode) {
|
||||
const sorted = [...rows];
|
||||
if (mode === "risk-desc") sorted.sort((a, b) => b.nonpass_rate - a.nonpass_rate);
|
||||
else if (mode === "risk-asc") sorted.sort((a, b) => a.nonpass_rate - b.nonpass_rate);
|
||||
else if (mode === "name") sorted.sort((a, b) => scorecardLabel(a).localeCompare(scorecardLabel(b)));
|
||||
else sorted.sort((a, b) => b.support_rounded - a.support_rounded);
|
||||
return sorted;
|
||||
}
|
||||
|
||||
function renderScorecardCards(rows) {
|
||||
const container = byId("cohort-cards");
|
||||
container.replaceChildren();
|
||||
for (const row of rows.slice(0, 18)) {
|
||||
const article = document.createElement("article");
|
||||
article.className = "cohort-card";
|
||||
const heading = document.createElement("h3");
|
||||
heading.textContent = scorecardLabel(row);
|
||||
const meta = document.createElement("div");
|
||||
meta.className = "cohort-card__meta";
|
||||
meta.textContent = `${compactNumber(row.support_rounded)} rounded support`;
|
||||
const bar = document.createElement("div");
|
||||
bar.className = "risk-bar";
|
||||
bar.setAttribute("aria-hidden", "true");
|
||||
const fill = document.createElement("span");
|
||||
fill.style.width = `${Math.min(100, row.nonpass_rate * 100)}%`;
|
||||
bar.append(fill);
|
||||
const value = document.createElement("div");
|
||||
value.className = "cohort-card__value";
|
||||
const strong = document.createElement("strong");
|
||||
strong.textContent = percent(row.nonpass_rate);
|
||||
const small = document.createElement("small");
|
||||
small.textContent = "Observed non-pass";
|
||||
value.append(strong, small);
|
||||
article.append(heading, meta, bar, value);
|
||||
container.append(article);
|
||||
}
|
||||
}
|
||||
|
||||
function updateExplorer() {
|
||||
if (!dashboardData) return;
|
||||
const query = byId("cohort-search").value.trim().toLowerCase();
|
||||
const sortMode = byId("result-sort").value;
|
||||
const matches = dashboardData.scorecards.rows.filter((row) =>
|
||||
scorecardLabel(row).toLowerCase().includes(query),
|
||||
);
|
||||
visibleScorecards = sortedScorecards(matches, sortMode);
|
||||
setText(
|
||||
"result-summary",
|
||||
`${visibleScorecards.length} supported cohort${visibleScorecards.length === 1 ? "" : "s"}; showing up to 18 cards and 12 chart rows.`,
|
||||
);
|
||||
byId("empty-results").hidden = visibleScorecards.length > 0;
|
||||
renderCohortDotPlot(byId("cohort-dot-plot"), visibleScorecards);
|
||||
renderScorecardCards(visibleScorecards);
|
||||
}
|
||||
|
||||
function initializeExplorer(data) {
|
||||
populateSelect("filter-county", data.filters.public_counties);
|
||||
populateSelect("filter-age", data.filters.age_bands, (value) => String(value));
|
||||
populateSelect(
|
||||
"filter-period",
|
||||
data.filters.periods.map((period) => period.year),
|
||||
(value) => String(value),
|
||||
);
|
||||
|
||||
// Current scorecards are make/model aggregates only. These planned controls
|
||||
// remain visible but disabled so the UI never implies unsupported slicing.
|
||||
disableUnsupportedFilter("filter-county", "Unavailable at current scorecard grain");
|
||||
disableUnsupportedFilter("filter-age", "Unavailable at current scorecard grain");
|
||||
disableUnsupportedFilter("filter-fuel", "Fuel not published in this bundle");
|
||||
disableUnsupportedFilter("filter-program", "Program not published in this bundle");
|
||||
disableUnsupportedFilter("filter-period", "Period not published in this scorecard");
|
||||
const adjusted = document.querySelector('input[name="risk-view"][value="adjusted"]');
|
||||
adjusted.disabled = true;
|
||||
adjusted.parentElement.title = "Model-adjusted cohort scorecards are not published.";
|
||||
|
||||
byId("cohort-search").addEventListener("input", updateExplorer);
|
||||
byId("result-sort").addEventListener("change", updateExplorer);
|
||||
byId("reset-filters").addEventListener("click", () => {
|
||||
byId("explorer-filters").reset();
|
||||
byId("cohort-search").value = "";
|
||||
byId("result-sort").value = "support";
|
||||
updateExplorer();
|
||||
});
|
||||
updateExplorer();
|
||||
}
|
||||
|
||||
function chooseDiagnostic(rows) {
|
||||
const partitionRank = { calibrate: 3, tune: 2, train: 1 };
|
||||
return [...rows].sort((left, right) => {
|
||||
const preferredLeft = left.model === "logistic_platt" ? 1 : 0;
|
||||
const preferredRight = right.model === "logistic_platt" ? 1 : 0;
|
||||
return (
|
||||
preferredRight - preferredLeft ||
|
||||
(partitionRank[right.partition] ?? 0) - (partitionRank[left.partition] ?? 0)
|
||||
);
|
||||
})[0];
|
||||
}
|
||||
|
||||
function renderMethods(data) {
|
||||
renderCoverageHeatmap(byId("coverage-heatmap"), data.coverage.rows);
|
||||
const diagnostic = chooseDiagnostic(data.diagnostics.rows);
|
||||
setText(
|
||||
"diagnostic-scope-label",
|
||||
diagnostic
|
||||
? `${diagnostic.partition === "calibrate" ? "Calibration cohort" : "Development diagnostics"} · ${diagnostic.model}`
|
||||
: "Development diagnostics · model unavailable",
|
||||
);
|
||||
setText("diagnostic-ap", diagnostic ? diagnostic.average_precision.toFixed(3) : "—");
|
||||
setText("diagnostic-brier", diagnostic ? diagnostic.brier.toFixed(3) : "—");
|
||||
setText("diagnostic-n", "Not published");
|
||||
renderNoCalibration(byId("calibration-chart"));
|
||||
}
|
||||
|
||||
function renderDashboard(data) {
|
||||
dashboardData = data;
|
||||
setPreviewBanner(data.manifest);
|
||||
renderOverview(data);
|
||||
initializeExplorer(data);
|
||||
renderMethods(data);
|
||||
setHeaderStatus(
|
||||
"ready",
|
||||
data.manifest.development_preview ? "Validated development aggregates" : "Validated public aggregates",
|
||||
);
|
||||
}
|
||||
|
||||
async function initialize() {
|
||||
initializeRouting();
|
||||
try {
|
||||
const data = await loadDashboardData();
|
||||
renderDashboard(data);
|
||||
} catch (error) {
|
||||
console.error("Dashboard data contract rejected the public assets.", error);
|
||||
setUnavailable(error);
|
||||
}
|
||||
}
|
||||
|
||||
initialize();
|
||||
|
||||
export { displayCategory, normalizeCounty, routeFromHash };
|
||||
281
dashboard/js/charts.js
Normal file
281
dashboard/js/charts.js
Normal file
@ -0,0 +1,281 @@
|
||||
const SVG_NS = "http://www.w3.org/2000/svg";
|
||||
|
||||
function escapeText(value) {
|
||||
return String(value)
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
}
|
||||
|
||||
function percent(value, digits = 1) {
|
||||
return new Intl.NumberFormat("en-US", {
|
||||
style: "percent",
|
||||
minimumFractionDigits: digits,
|
||||
maximumFractionDigits: digits,
|
||||
}).format(value);
|
||||
}
|
||||
|
||||
function compactNumber(value) {
|
||||
return new Intl.NumberFormat("en-US", {
|
||||
notation: value >= 10_000 ? "compact" : "standard",
|
||||
maximumFractionDigits: 1,
|
||||
}).format(value);
|
||||
}
|
||||
|
||||
function emptyChart(container, message) {
|
||||
container.innerHTML = `<div class="chart-empty"><p>${escapeText(message)}</p></div>`;
|
||||
container.setAttribute("aria-label", message);
|
||||
}
|
||||
|
||||
function aggregateOverview(rows) {
|
||||
const groups = new Map();
|
||||
for (const row of rows) {
|
||||
const key = `${row.year}-Q${row.quarter}`;
|
||||
const current = groups.get(key) ?? {
|
||||
label: `${row.year} Q${row.quarter}`,
|
||||
year: row.year,
|
||||
quarter: row.quarter,
|
||||
support: 0,
|
||||
weightedRisk: 0,
|
||||
};
|
||||
current.support += row.support_rounded;
|
||||
current.weightedRisk += row.support_rounded * row.nonpass_rate;
|
||||
groups.set(key, current);
|
||||
}
|
||||
return [...groups.values()]
|
||||
.map((group) => ({
|
||||
...group,
|
||||
risk: group.support > 0 ? group.weightedRisk / group.support : 0,
|
||||
}))
|
||||
.sort((left, right) => left.year - right.year || left.quarter - right.quarter);
|
||||
}
|
||||
|
||||
export function renderOutcomeTrend(container, rows) {
|
||||
const points = aggregateOverview(rows);
|
||||
if (points.length < 2) {
|
||||
emptyChart(container, "Not enough approved periods to draw a trend.");
|
||||
return;
|
||||
}
|
||||
const width = 920;
|
||||
const height = 330;
|
||||
const margin = { top: 24, right: 22, bottom: 54, left: 58 };
|
||||
const plotWidth = width - margin.left - margin.right;
|
||||
const plotHeight = height - margin.top - margin.bottom;
|
||||
const maxRisk = Math.max(0.05, ...points.map((point) => point.risk));
|
||||
const yMax = Math.min(1, Math.ceil(maxRisk * 20) / 20);
|
||||
const x = (index) => margin.left + (index / (points.length - 1)) * plotWidth;
|
||||
const y = (value) => margin.top + plotHeight - (value / yMax) * plotHeight;
|
||||
const path = points
|
||||
.map((point, index) => `${index === 0 ? "M" : "L"}${x(index).toFixed(1)},${y(point.risk).toFixed(1)}`)
|
||||
.join(" ");
|
||||
|
||||
const yTicks = Array.from({ length: 5 }, (_, index) => (index / 4) * yMax);
|
||||
const tickEvery = Math.max(1, Math.ceil(points.length / 8));
|
||||
const labelled = points.filter(
|
||||
(_point, index) => index % tickEvery === 0 || index === points.length - 1,
|
||||
);
|
||||
const description = points
|
||||
.map((point) => `${point.label}: ${percent(point.risk)}`)
|
||||
.join("; ");
|
||||
|
||||
container.innerHTML = `
|
||||
<svg viewBox="0 0 ${width} ${height}" aria-hidden="true" focusable="false">
|
||||
${yTicks
|
||||
.map(
|
||||
(tick) => `
|
||||
<line class="grid-line" x1="${margin.left}" x2="${width - margin.right}" y1="${y(tick)}" y2="${y(tick)}"></line>
|
||||
<text class="axis-label" x="${margin.left - 10}" y="${y(tick) + 4}" text-anchor="end">${percent(tick, 0)}</text>`,
|
||||
)
|
||||
.join("")}
|
||||
<line class="axis-line" x1="${margin.left}" x2="${width - margin.right}" y1="${height - margin.bottom}" y2="${height - margin.bottom}"></line>
|
||||
<path class="chart-line" d="${path}"></path>
|
||||
${points
|
||||
.map(
|
||||
(point, index) => `
|
||||
<circle class="chart-dot" cx="${x(index)}" cy="${y(point.risk)}" r="4">
|
||||
<title>${escapeText(point.label)}: ${percent(point.risk)} non-pass (${compactNumber(point.support)} rounded support)</title>
|
||||
</circle>`,
|
||||
)
|
||||
.join("")}
|
||||
${labelled
|
||||
.map((point) => {
|
||||
const index = points.indexOf(point);
|
||||
return `<text class="axis-label" x="${x(index)}" y="${height - 24}" text-anchor="middle">${escapeText(point.label.replace(" ", "\u00a0"))}</text>`;
|
||||
})
|
||||
.join("")}
|
||||
<text class="axis-label" transform="translate(16 ${height / 2}) rotate(-90)" text-anchor="middle">Non-pass rate</text>
|
||||
</svg>`;
|
||||
container.setAttribute(
|
||||
"aria-label",
|
||||
`Quarterly aggregate non-pass trend. ${description}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function renderAgeRisk(container, rows) {
|
||||
if (!rows.length) {
|
||||
emptyChart(container, "No supported vehicle-age bands are available.");
|
||||
return;
|
||||
}
|
||||
const points = [...rows];
|
||||
const width = 600;
|
||||
const height = 300;
|
||||
const margin = { top: 24, right: 20, bottom: 58, left: 52 };
|
||||
const plotWidth = width - margin.left - margin.right;
|
||||
const plotHeight = height - margin.top - margin.bottom;
|
||||
const maxRisk = Math.max(0.05, ...points.map((point) => point.nonpass_rate));
|
||||
const yMax = Math.min(1, Math.ceil(maxRisk * 20) / 20);
|
||||
const denominator = Math.max(1, points.length - 1);
|
||||
const x = (index) => margin.left + (index / denominator) * plotWidth;
|
||||
const y = (value) => margin.top + plotHeight - (value / yMax) * plotHeight;
|
||||
const path = points
|
||||
.map((point, index) => `${index === 0 ? "M" : "L"}${x(index)},${y(point.nonpass_rate)}`)
|
||||
.join(" ");
|
||||
|
||||
container.innerHTML = `
|
||||
<svg viewBox="0 0 ${width} ${height}" aria-hidden="true" focusable="false">
|
||||
${[0, 0.25, 0.5, 0.75, 1]
|
||||
.map((ratio) => {
|
||||
const tick = ratio * yMax;
|
||||
return `<line class="grid-line" x1="${margin.left}" x2="${width - margin.right}" y1="${y(tick)}" y2="${y(tick)}"></line>
|
||||
<text class="axis-label" x="${margin.left - 8}" y="${y(tick) + 4}" text-anchor="end">${percent(tick, 0)}</text>`;
|
||||
})
|
||||
.join("")}
|
||||
<path class="chart-line" d="${path}"></path>
|
||||
${points
|
||||
.map(
|
||||
(point, index) => `
|
||||
<circle class="chart-dot" cx="${x(index)}" cy="${y(point.nonpass_rate)}" r="5">
|
||||
<title>${escapeText(point.age_band)}: ${percent(point.nonpass_rate)} (${compactNumber(point.support_rounded)} rounded support)</title>
|
||||
</circle>
|
||||
<text class="axis-label" x="${x(index)}" y="${height - 25}" text-anchor="middle">${escapeText(point.age_band)}</text>`,
|
||||
)
|
||||
.join("")}
|
||||
</svg>`;
|
||||
container.setAttribute(
|
||||
"aria-label",
|
||||
`Non-pass risk by vehicle-age band. ${points
|
||||
.map((point) => `${point.age_band}: ${percent(point.nonpass_rate)}`)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function renderCohortDotPlot(container, rows) {
|
||||
const points = rows.slice(0, 12);
|
||||
if (!points.length) {
|
||||
emptyChart(container, "No supported cohorts match the current filters.");
|
||||
return;
|
||||
}
|
||||
const width = 820;
|
||||
const rowHeight = 34;
|
||||
const margin = { top: 24, right: 60, bottom: 42, left: 220 };
|
||||
const height = margin.top + margin.bottom + points.length * rowHeight;
|
||||
const plotWidth = width - margin.left - margin.right;
|
||||
const maxRisk = Math.max(0.05, ...points.map((point) => point.nonpass_rate));
|
||||
const xMax = Math.min(1, Math.ceil(maxRisk * 20) / 20);
|
||||
const x = (value) => margin.left + (value / xMax) * plotWidth;
|
||||
|
||||
container.innerHTML = `
|
||||
<svg viewBox="0 0 ${width} ${height}" aria-hidden="true" focusable="false">
|
||||
${[0, 0.25, 0.5, 0.75, 1]
|
||||
.map((ratio) => {
|
||||
const value = ratio * xMax;
|
||||
return `<line class="grid-line" x1="${x(value)}" x2="${x(value)}" y1="${margin.top - 8}" y2="${height - margin.bottom + 5}"></line>
|
||||
<text class="axis-label" x="${x(value)}" y="${height - 15}" text-anchor="middle">${percent(value, 0)}</text>`;
|
||||
})
|
||||
.join("")}
|
||||
${points
|
||||
.map((point, index) => {
|
||||
const y = margin.top + index * rowHeight + rowHeight / 2;
|
||||
const label = `${point.prior_make} ${point.prior_model}`;
|
||||
return `
|
||||
<text class="axis-label" x="${margin.left - 12}" y="${y + 4}" text-anchor="end">${escapeText(label)}</text>
|
||||
<line x1="${margin.left}" x2="${x(point.nonpass_rate)}" y1="${y}" y2="${y}" stroke="var(--red-100)" stroke-width="8" stroke-linecap="round"></line>
|
||||
<circle class="chart-dot" cx="${x(point.nonpass_rate)}" cy="${y}" r="5"><title>${escapeText(label)}: ${percent(point.nonpass_rate)}, ${compactNumber(point.support_rounded)} rounded support</title></circle>
|
||||
<text class="axis-label" x="${x(point.nonpass_rate) + 10}" y="${y + 4}">${percent(point.nonpass_rate)}</text>`;
|
||||
})
|
||||
.join("")}
|
||||
</svg>`;
|
||||
container.setAttribute(
|
||||
"aria-label",
|
||||
`Ranked supported cohort non-pass risk. ${points
|
||||
.map((point) => `${point.prior_make} ${point.prior_model}: ${percent(point.nonpass_rate)}`)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const COUNTY_POINTS = {
|
||||
cache: [154, 40],
|
||||
weber: [137, 86],
|
||||
davis: [126, 108],
|
||||
salt_lake: [133, 133],
|
||||
"salt lake": [133, 133],
|
||||
utah: [140, 170],
|
||||
};
|
||||
|
||||
export function renderUtahCoverage(container, coveredCounties) {
|
||||
const normalized = new Set(coveredCounties.map((county) => county.toLowerCase()));
|
||||
const points = [...normalized]
|
||||
.map((county) => ({ county, coordinates: COUNTY_POINTS[county] }))
|
||||
.filter((item) => item.coordinates);
|
||||
container.innerHTML = `
|
||||
<svg viewBox="0 0 270 300" aria-hidden="true" focusable="false">
|
||||
<path d="M79 18h101v54l18 18v183H52V116l27-27V18Z" fill="var(--gray-100)" stroke="var(--gray-300)" stroke-width="3"></path>
|
||||
<path d="M80 20h98v53l17 18v179H55V117l25-27V20Z" fill="none" stroke="var(--sand-200)" stroke-width="1.5" stroke-dasharray="4 5"></path>
|
||||
${points
|
||||
.map(
|
||||
({ county, coordinates }) => `
|
||||
<circle cx="${coordinates[0]}" cy="${coordinates[1]}" r="8" fill="var(--teal-700)" stroke="var(--paper)" stroke-width="3"><title>${escapeText(county.replace("_", " "))} feed available</title></circle>`,
|
||||
)
|
||||
.join("")}
|
||||
<text x="135" y="288" text-anchor="middle" class="axis-label">Participating feeds highlighted</text>
|
||||
</svg>`;
|
||||
container.setAttribute(
|
||||
"aria-label",
|
||||
coveredCounties.length
|
||||
? `Utah feed coverage includes ${coveredCounties.join(", ")}. Other counties are unavailable.`
|
||||
: "No county feed coverage is available.",
|
||||
);
|
||||
}
|
||||
|
||||
export function renderCoverageHeatmap(container, rows) {
|
||||
if (!rows.length) {
|
||||
emptyChart(container, "No source-era coverage rows are available.");
|
||||
return;
|
||||
}
|
||||
const years = [...new Set(rows.map((row) => row.year))].sort((a, b) => a - b);
|
||||
const eras = [...new Set(rows.map((row) => row.source_era))].sort();
|
||||
const lookup = new Map(rows.map((row) => [`${row.source_era}-${row.year}`, row]));
|
||||
const maxSupport = Math.max(1, ...rows.map((row) => row.support_rounded));
|
||||
const columns = `90px repeat(${years.length}, minmax(34px, 1fr))`;
|
||||
container.innerHTML = `
|
||||
<div class="heatmap-grid" style="grid-template-columns:${columns}">
|
||||
<span></span>${years.map((year) => `<span class="heatmap-label">${year}</span>`).join("")}
|
||||
${eras
|
||||
.map(
|
||||
(era) => `<span class="heatmap-label">${escapeText(era)}</span>${years
|
||||
.map((year) => {
|
||||
const row = lookup.get(`${era}-${year}`);
|
||||
if (!row) return `<span class="heatmap-cell" title="${escapeText(era)} ${year}: unavailable"></span>`;
|
||||
const alpha = 0.15 + 0.75 * Math.sqrt(row.support_rounded / maxSupport);
|
||||
return `<span class="heatmap-cell" style="background:rgb(25 116 119 / ${alpha.toFixed(2)})" title="${escapeText(era)} ${year}: ${compactNumber(row.support_rounded)} rounded support; ${percent(row.labeled_rate)} labeled"></span>`;
|
||||
})
|
||||
.join("")}`,
|
||||
)
|
||||
.join("")}
|
||||
</div>`;
|
||||
container.setAttribute(
|
||||
"aria-label",
|
||||
`Source-era coverage from ${years[0]} through ${years[years.length - 1]} for ${eras.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
|
||||
export function renderNoCalibration(container) {
|
||||
emptyChart(
|
||||
container,
|
||||
"Calibration-bin data is not published in this development bundle. No calibration curve is shown.",
|
||||
);
|
||||
}
|
||||
|
||||
export { aggregateOverview, compactNumber, percent };
|
||||
574
dashboard/js/data.js
Normal file
574
dashboard/js/data.js
Normal file
@ -0,0 +1,574 @@
|
||||
const SCHEMA_VERSION = "dashboard_data_v1";
|
||||
const APPROVED_PARTITIONS = Object.freeze(["train", "tune", "calibrate"]);
|
||||
const ENVELOPE_KEYS = Object.freeze([
|
||||
"development_preview",
|
||||
"population_estimate_allowed",
|
||||
"schema_version",
|
||||
]);
|
||||
|
||||
export const REQUIRED_ASSETS = Object.freeze({
|
||||
manifest: "data_manifest.json",
|
||||
overview: "overview_period_county.json",
|
||||
ageRisk: "age_risk_curve.json",
|
||||
scorecards: "cohort_scorecard.json",
|
||||
diagnostics: "model_diagnostics.json",
|
||||
coverage: "coverage_quality.json",
|
||||
filters: "filter_catalog.json",
|
||||
shaManifest: "sha256_manifest.json",
|
||||
});
|
||||
|
||||
const SENSITIVE_KEY_PARTS = new Set([
|
||||
"address",
|
||||
"certificate",
|
||||
"email",
|
||||
"internal",
|
||||
"ip",
|
||||
"owner",
|
||||
"pid",
|
||||
"plate",
|
||||
"raw",
|
||||
"session",
|
||||
"station",
|
||||
"token",
|
||||
"user",
|
||||
"vin",
|
||||
"zip",
|
||||
]);
|
||||
|
||||
const ASSET_VALIDATORS = {
|
||||
overview: (value) => validateRows(value, validateOverviewRow, "overview_period_county"),
|
||||
ageRisk: (value) => validateRows(value, validateAgeRiskRow, "age_risk_curve"),
|
||||
scorecards: (value) => validateRows(value, validateScorecardRow, "cohort_scorecard"),
|
||||
diagnostics: (value) => validateRows(value, validateDiagnosticRow, "model_diagnostics"),
|
||||
coverage: (value) => validateRows(value, validateCoverageRow, "coverage_quality"),
|
||||
filters: validateFilterCatalog,
|
||||
shaManifest: validateShaManifest,
|
||||
};
|
||||
|
||||
export class DataContractError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "DataContractError";
|
||||
}
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
||||
const prototype = Object.getPrototypeOf(value);
|
||||
return prototype === Object.prototype || prototype === null;
|
||||
}
|
||||
|
||||
function requirePlainObject(value, label) {
|
||||
if (!isPlainObject(value)) {
|
||||
throw new DataContractError(`${label} must be a JSON object.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireExactKeys(value, expectedKeys, label) {
|
||||
const observed = Object.keys(value).sort();
|
||||
const expected = [...expectedKeys].sort();
|
||||
if (JSON.stringify(observed) !== JSON.stringify(expected)) {
|
||||
throw new DataContractError(`${label} does not match the approved field schema.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireExactStringSet(value, expectedValues, label) {
|
||||
validateStringArray(value, label);
|
||||
const observed = [...value].sort();
|
||||
const expected = [...expectedValues].sort();
|
||||
if (JSON.stringify(observed) !== JSON.stringify(expected)) {
|
||||
throw new DataContractError(`${label} does not match the approved values.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireBoolean(value, label) {
|
||||
if (typeof value !== "boolean") {
|
||||
throw new DataContractError(`${label} must be true or false.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireString(value, label) {
|
||||
if (typeof value !== "string" || value.trim() === "") {
|
||||
throw new DataContractError(`${label} must be a non-empty string.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireSafeModelVersion(value, label) {
|
||||
requireString(value, label);
|
||||
if (!/^[A-Za-z0-9._-]{1,64}$/.test(value)) {
|
||||
throw new DataContractError(`${label} is not an approved model version.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireNumber(value, label, { min = -Infinity, max = Infinity } = {}) {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) {
|
||||
throw new DataContractError(`${label} must be a finite number in the approved range.`);
|
||||
}
|
||||
}
|
||||
|
||||
function requireInteger(value, label, { min = -Infinity, max = Infinity } = {}) {
|
||||
requireNumber(value, label, { min, max });
|
||||
if (!Number.isInteger(value)) {
|
||||
throw new DataContractError(`${label} must be an integer.`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateEnvelope(value, label) {
|
||||
requirePlainObject(value, label);
|
||||
if (value.schema_version !== SCHEMA_VERSION) {
|
||||
throw new DataContractError(`${label} uses an unsupported schema version.`);
|
||||
}
|
||||
requireBoolean(value.development_preview, `${label}.development_preview`);
|
||||
requireBoolean(
|
||||
value.population_estimate_allowed,
|
||||
`${label}.population_estimate_allowed`,
|
||||
);
|
||||
rejectSensitiveKeys(value, label);
|
||||
}
|
||||
|
||||
function validateRows(value, rowValidator, label) {
|
||||
validateEnvelope(value, label);
|
||||
requireExactKeys(value, [...ENVELOPE_KEYS, "rows"], label);
|
||||
if (!Array.isArray(value.rows)) {
|
||||
throw new DataContractError(`${label}.rows must be an array.`);
|
||||
}
|
||||
value.rows.forEach((row, index) => {
|
||||
requirePlainObject(row, `${label}.rows[${index}]`);
|
||||
rowValidator(row, `${label}.rows[${index}]`);
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateOverviewRow(row, label) {
|
||||
requireExactKeys(
|
||||
row,
|
||||
["year", "quarter", "public_county", "support_rounded", "nonpass_rate"],
|
||||
label,
|
||||
);
|
||||
requireInteger(row.year, `${label}.year`, { min: 2010, max: 2100 });
|
||||
requireInteger(row.quarter, `${label}.quarter`, { min: 1, max: 4 });
|
||||
requireString(row.public_county, `${label}.public_county`);
|
||||
requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 });
|
||||
requireNumber(row.nonpass_rate, `${label}.nonpass_rate`, { min: 0, max: 1 });
|
||||
}
|
||||
|
||||
function validateAgeRiskRow(row, label) {
|
||||
requireExactKeys(row, ["age_band", "support_rounded", "nonpass_rate"], label);
|
||||
requireString(row.age_band, `${label}.age_band`);
|
||||
requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 });
|
||||
requireNumber(row.nonpass_rate, `${label}.nonpass_rate`, { min: 0, max: 1 });
|
||||
}
|
||||
|
||||
function validateScorecardRow(row, label) {
|
||||
requireExactKeys(
|
||||
row,
|
||||
["prior_make", "prior_model", "support_rounded", "nonpass_rate"],
|
||||
label,
|
||||
);
|
||||
requireString(row.prior_make, `${label}.prior_make`);
|
||||
requireString(row.prior_model, `${label}.prior_model`);
|
||||
requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 });
|
||||
requireNumber(row.nonpass_rate, `${label}.nonpass_rate`, { min: 0, max: 1 });
|
||||
}
|
||||
|
||||
function validateDiagnosticRow(row, label) {
|
||||
requireExactKeys(
|
||||
row,
|
||||
[
|
||||
"model",
|
||||
"partition",
|
||||
"average_precision",
|
||||
"brier",
|
||||
"log_loss",
|
||||
"roc_auc",
|
||||
"top_10_capture",
|
||||
],
|
||||
label,
|
||||
);
|
||||
requireString(row.model, `${label}.model`);
|
||||
requireString(row.partition, `${label}.partition`);
|
||||
if (!APPROVED_PARTITIONS.includes(row.partition)) {
|
||||
throw new DataContractError(`${label}.partition is not approved for this public schema.`);
|
||||
}
|
||||
for (const metric of [
|
||||
"average_precision",
|
||||
"brier",
|
||||
"log_loss",
|
||||
"roc_auc",
|
||||
"top_10_capture",
|
||||
]) {
|
||||
requireNumber(row[metric], `${label}.${metric}`, { min: 0 });
|
||||
}
|
||||
if (row.average_precision > 1 || row.brier > 1 || row.roc_auc > 1 || row.top_10_capture > 1) {
|
||||
throw new DataContractError(`${label} contains a probability metric above 1.`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateCoverageRow(row, label) {
|
||||
requireExactKeys(
|
||||
row,
|
||||
[
|
||||
"year",
|
||||
"source_era",
|
||||
"support_rounded",
|
||||
"labeled_rate",
|
||||
"overall_result_share",
|
||||
"utah_obd_proxy_share",
|
||||
],
|
||||
label,
|
||||
);
|
||||
requireInteger(row.year, `${label}.year`, { min: 2010, max: 2100 });
|
||||
requireString(row.source_era, `${label}.source_era`);
|
||||
requireInteger(row.support_rounded, `${label}.support_rounded`, { min: 0 });
|
||||
for (const metric of ["labeled_rate", "overall_result_share", "utah_obd_proxy_share"]) {
|
||||
requireNumber(row[metric], `${label}.${metric}`, { min: 0, max: 1 });
|
||||
}
|
||||
}
|
||||
|
||||
function validateStringArray(value, label) {
|
||||
if (!Array.isArray(value)) throw new DataContractError(`${label} must be an array.`);
|
||||
value.forEach((item, index) => requireString(item, `${label}[${index}]`));
|
||||
}
|
||||
|
||||
function validateFilterCatalog(value) {
|
||||
const label = "filter_catalog";
|
||||
validateEnvelope(value, label);
|
||||
requireExactKeys(
|
||||
value,
|
||||
[
|
||||
...ENVELOPE_KEYS,
|
||||
"public_counties",
|
||||
"age_bands",
|
||||
"prior_make_models",
|
||||
"periods",
|
||||
"models",
|
||||
"partitions",
|
||||
],
|
||||
label,
|
||||
);
|
||||
validateStringArray(value.public_counties, `${label}.public_counties`);
|
||||
validateStringArray(value.age_bands, `${label}.age_bands`);
|
||||
validateStringArray(value.models, `${label}.models`);
|
||||
requireExactStringSet(value.partitions, APPROVED_PARTITIONS, `${label}.partitions`);
|
||||
if (!Array.isArray(value.prior_make_models)) {
|
||||
throw new DataContractError(`${label}.prior_make_models must be an array.`);
|
||||
}
|
||||
value.prior_make_models.forEach((row, index) => {
|
||||
requirePlainObject(row, `${label}.prior_make_models[${index}]`);
|
||||
requireExactKeys(
|
||||
row,
|
||||
["prior_make", "prior_model"],
|
||||
`${label}.prior_make_models[${index}]`,
|
||||
);
|
||||
requireString(row.prior_make, `${label}.prior_make_models[${index}].prior_make`);
|
||||
requireString(row.prior_model, `${label}.prior_make_models[${index}].prior_model`);
|
||||
});
|
||||
if (!Array.isArray(value.periods)) {
|
||||
throw new DataContractError(`${label}.periods must be an array.`);
|
||||
}
|
||||
value.periods.forEach((period, index) => {
|
||||
requirePlainObject(period, `${label}.periods[${index}]`);
|
||||
requireExactKeys(period, ["year", "quarters"], `${label}.periods[${index}]`);
|
||||
requireInteger(period.year, `${label}.periods[${index}].year`, {
|
||||
min: 2010,
|
||||
max: 2100,
|
||||
});
|
||||
if (!Array.isArray(period.quarters) || period.quarters.length === 0) {
|
||||
throw new DataContractError(`${label}.periods[${index}].quarters must be non-empty.`);
|
||||
}
|
||||
period.quarters.forEach((quarter, quarterIndex) =>
|
||||
requireInteger(quarter, `${label}.periods[${index}].quarters[${quarterIndex}]`, {
|
||||
min: 1,
|
||||
max: 4,
|
||||
}),
|
||||
);
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateShaManifest(value) {
|
||||
const label = "sha256_manifest";
|
||||
validateEnvelope(value, label);
|
||||
requireExactKeys(value, [...ENVELOPE_KEYS, "files"], label);
|
||||
if (!Array.isArray(value.files)) {
|
||||
throw new DataContractError(`${label}.files must be an array.`);
|
||||
}
|
||||
const expectedNames = Object.entries(REQUIRED_ASSETS)
|
||||
.filter(([name]) => name !== "shaManifest")
|
||||
.map(([, filename]) => filename)
|
||||
.sort();
|
||||
const observedNames = [];
|
||||
for (const [index, file] of value.files.entries()) {
|
||||
requirePlainObject(file, `${label}.files[${index}]`);
|
||||
requireExactKeys(file, ["name", "sha256"], `${label}.files[${index}]`);
|
||||
requireString(file.name, `${label}.files[${index}].name`);
|
||||
requireString(file.sha256, `${label}.files[${index}].sha256`);
|
||||
if (!/^[0-9a-f]{64}$/.test(file.sha256)) {
|
||||
throw new DataContractError(`${label}.files[${index}].sha256 is invalid.`);
|
||||
}
|
||||
observedNames.push(file.name);
|
||||
}
|
||||
observedNames.sort();
|
||||
if (JSON.stringify(observedNames) !== JSON.stringify(expectedNames)) {
|
||||
throw new DataContractError(`${label} does not cover the exact approved asset set.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function rejectSensitiveKeys(value, label, seen = new WeakSet()) {
|
||||
if (value === null || typeof value !== "object") return;
|
||||
if (seen.has(value)) return;
|
||||
seen.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item, index) => rejectSensitiveKeys(item, `${label}[${index}]`, seen));
|
||||
return;
|
||||
}
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
const normalizedParts = key.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
||||
if (normalizedParts.some((part) => SENSITIVE_KEY_PARTS.has(part))) {
|
||||
throw new DataContractError(`${label} contains a field outside the public data contract.`);
|
||||
}
|
||||
rejectSensitiveKeys(child, `${label}.${key}`, seen);
|
||||
}
|
||||
}
|
||||
|
||||
export function validateAssetSet(rawAssets) {
|
||||
requirePlainObject(rawAssets, "asset set");
|
||||
requireExactKeys(rawAssets, Object.keys(REQUIRED_ASSETS), "asset set");
|
||||
const manifest = rawAssets.manifest;
|
||||
validateEnvelope(manifest, "data_manifest");
|
||||
requireExactKeys(
|
||||
manifest,
|
||||
[
|
||||
...ENVELOPE_KEYS,
|
||||
"assets",
|
||||
"data_scope",
|
||||
"definitions",
|
||||
"model_versions",
|
||||
"release_id",
|
||||
],
|
||||
"data_manifest",
|
||||
);
|
||||
if (Object.hasOwn(manifest, "rows")) {
|
||||
throw new DataContractError("data_manifest must not contain row data.");
|
||||
}
|
||||
requirePlainObject(manifest.data_scope, "data_manifest.data_scope");
|
||||
requirePlainObject(manifest.definitions, "data_manifest.definitions");
|
||||
requireString(manifest.release_id, "data_manifest.release_id");
|
||||
if (!/^[0-9a-f]{64}$/.test(manifest.release_id)) {
|
||||
throw new DataContractError("data_manifest.release_id is invalid.");
|
||||
}
|
||||
if (!Array.isArray(manifest.model_versions) || manifest.model_versions.length === 0) {
|
||||
throw new DataContractError("data_manifest.model_versions must be a non-empty array.");
|
||||
}
|
||||
manifest.model_versions.forEach((version, index) =>
|
||||
requireSafeModelVersion(version, `data_manifest.model_versions[${index}]`),
|
||||
);
|
||||
if (
|
||||
JSON.stringify(manifest.model_versions) !==
|
||||
JSON.stringify([...new Set(manifest.model_versions)].sort())
|
||||
) {
|
||||
throw new DataContractError("data_manifest.model_versions must be sorted and unique.");
|
||||
}
|
||||
requireExactKeys(
|
||||
manifest.data_scope,
|
||||
["first_year", "last_year", "model_names", "partitions"],
|
||||
"data_manifest.data_scope",
|
||||
);
|
||||
requireInteger(manifest.data_scope.first_year, "data_manifest.data_scope.first_year", {
|
||||
min: 2010,
|
||||
max: 2100,
|
||||
});
|
||||
requireInteger(manifest.data_scope.last_year, "data_manifest.data_scope.last_year", {
|
||||
min: manifest.data_scope.first_year,
|
||||
max: 2100,
|
||||
});
|
||||
validateStringArray(manifest.data_scope.model_names, "data_manifest.data_scope.model_names");
|
||||
requireExactStringSet(
|
||||
manifest.data_scope.partitions,
|
||||
APPROVED_PARTITIONS,
|
||||
"data_manifest.data_scope.partitions",
|
||||
);
|
||||
requireExactKeys(
|
||||
manifest.definitions,
|
||||
[
|
||||
"episode_gap_days",
|
||||
"locked_test_metrics_published",
|
||||
"support_rounding",
|
||||
"suppression_min_nonpass",
|
||||
"suppression_min_pass",
|
||||
"suppression_min_support",
|
||||
"suppression_min_distinct_vehicles",
|
||||
"suppression_min_distinct_pass_vehicles",
|
||||
"suppression_min_distinct_nonpass_vehicles",
|
||||
"target",
|
||||
],
|
||||
"data_manifest.definitions",
|
||||
);
|
||||
requireInteger(manifest.definitions.episode_gap_days, "data_manifest.definitions.episode_gap_days", {
|
||||
min: 1,
|
||||
});
|
||||
requireBoolean(
|
||||
manifest.definitions.locked_test_metrics_published,
|
||||
"data_manifest.definitions.locked_test_metrics_published",
|
||||
);
|
||||
if (manifest.definitions.locked_test_metrics_published !== false) {
|
||||
throw new DataContractError("Locked-test metrics are not approved for this public preview.");
|
||||
}
|
||||
for (const key of [
|
||||
"support_rounding",
|
||||
"suppression_min_nonpass",
|
||||
"suppression_min_pass",
|
||||
"suppression_min_support",
|
||||
"suppression_min_distinct_vehicles",
|
||||
"suppression_min_distinct_pass_vehicles",
|
||||
"suppression_min_distinct_nonpass_vehicles",
|
||||
]) {
|
||||
requireInteger(manifest.definitions[key], `data_manifest.definitions.${key}`, { min: 1 });
|
||||
}
|
||||
requireString(manifest.definitions.target, "data_manifest.definitions.target");
|
||||
if (!Array.isArray(manifest.assets)) {
|
||||
throw new DataContractError("data_manifest.assets must be an array.");
|
||||
}
|
||||
validateStringArray(manifest.assets, "data_manifest.assets");
|
||||
const expectedManifestAssets = Object.entries(REQUIRED_ASSETS)
|
||||
.filter(([name]) => !["manifest", "shaManifest"].includes(name))
|
||||
.map(([, filename]) => filename)
|
||||
.sort();
|
||||
const declaredAssets = [...manifest.assets].sort();
|
||||
if (JSON.stringify(declaredAssets) !== JSON.stringify(expectedManifestAssets)) {
|
||||
throw new DataContractError("data_manifest.assets does not match the approved asset set.");
|
||||
}
|
||||
|
||||
const validated = { manifest };
|
||||
for (const [name, validator] of Object.entries(ASSET_VALIDATORS)) {
|
||||
if (!Object.hasOwn(rawAssets, name)) {
|
||||
throw new DataContractError(`A required approved aggregate is missing: ${name}.`);
|
||||
}
|
||||
validated[name] = validator(rawAssets[name]);
|
||||
if (
|
||||
validated[name].development_preview !== manifest.development_preview ||
|
||||
validated[name].population_estimate_allowed !== manifest.population_estimate_allowed
|
||||
) {
|
||||
throw new DataContractError(`${name} publication flags disagree with data_manifest.`);
|
||||
}
|
||||
}
|
||||
requireExactStringSet(
|
||||
validated.filters.models,
|
||||
manifest.data_scope.model_names,
|
||||
"filter_catalog.models",
|
||||
);
|
||||
for (const [index, row] of validated.diagnostics.rows.entries()) {
|
||||
if (!validated.filters.models.includes(row.model)) {
|
||||
throw new DataContractError(`model_diagnostics.rows[${index}].model is not cataloged.`);
|
||||
}
|
||||
}
|
||||
const minimumSupport = manifest.definitions.suppression_min_support;
|
||||
const supportRounding = manifest.definitions.support_rounding;
|
||||
for (const name of ["overview", "ageRisk", "scorecards", "coverage"]) {
|
||||
for (const [index, row] of validated[name].rows.entries()) {
|
||||
if (
|
||||
row.support_rounded < minimumSupport ||
|
||||
row.support_rounded % supportRounding !== 0
|
||||
) {
|
||||
throw new DataContractError(
|
||||
`${name}.rows[${index}].support_rounded violates the publication thresholds.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Object.freeze(validated);
|
||||
}
|
||||
|
||||
async function fetchBytes(path, fetchImplementation) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImplementation(path, {
|
||||
cache: "no-store",
|
||||
credentials: "same-origin",
|
||||
});
|
||||
} catch {
|
||||
throw new DataContractError("Approved dashboard assets could not be reached.");
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new DataContractError("One or more approved dashboard assets are unavailable.");
|
||||
}
|
||||
try {
|
||||
return await response.arrayBuffer();
|
||||
} catch {
|
||||
throw new DataContractError("An approved dashboard asset could not be read safely.");
|
||||
}
|
||||
}
|
||||
|
||||
function parseJsonBytes(bytes, label) {
|
||||
if (typeof TextDecoder !== "function") {
|
||||
throw new DataContractError("This browser cannot decode dashboard assets safely.");
|
||||
}
|
||||
try {
|
||||
return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
|
||||
} catch {
|
||||
throw new DataContractError(`${label} is not valid UTF-8 JSON.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function sha256Hex(bytes, cryptoImplementation) {
|
||||
if (!cryptoImplementation?.subtle || typeof cryptoImplementation.subtle.digest !== "function") {
|
||||
throw new DataContractError("This browser cannot verify dashboard asset integrity.");
|
||||
}
|
||||
let digest;
|
||||
try {
|
||||
digest = await cryptoImplementation.subtle.digest("SHA-256", bytes);
|
||||
} catch {
|
||||
throw new DataContractError("Dashboard asset integrity verification failed.");
|
||||
}
|
||||
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
export async function loadDashboardData({
|
||||
basePath = "./public/data/",
|
||||
fetchImplementation = globalThis.fetch,
|
||||
cryptoImplementation = globalThis.crypto,
|
||||
} = {}) {
|
||||
if (typeof fetchImplementation !== "function") {
|
||||
throw new DataContractError("This browser cannot load dashboard assets safely.");
|
||||
}
|
||||
const baseUrl = new URL(basePath, globalThis.location?.href ?? "http://local/");
|
||||
const shaBytes = await fetchBytes(
|
||||
new URL(REQUIRED_ASSETS.shaManifest, baseUrl),
|
||||
fetchImplementation,
|
||||
);
|
||||
const shaManifest = validateShaManifest(
|
||||
parseJsonBytes(shaBytes, "The checksum manifest"),
|
||||
);
|
||||
const expectedDigests = new Map(
|
||||
shaManifest.files.map((entry) => [entry.name, entry.sha256]),
|
||||
);
|
||||
const assetEntries = Object.entries(REQUIRED_ASSETS).filter(
|
||||
([name]) => name !== "shaManifest",
|
||||
);
|
||||
const rawEntries = await Promise.all(
|
||||
assetEntries.map(async ([name, filename]) => [
|
||||
name,
|
||||
filename,
|
||||
await fetchBytes(new URL(filename, baseUrl), fetchImplementation),
|
||||
]),
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
rawEntries.map(async ([, filename, bytes]) => {
|
||||
const observed = await sha256Hex(bytes, cryptoImplementation);
|
||||
if (observed !== expectedDigests.get(filename)) {
|
||||
throw new DataContractError(`Integrity verification failed for ${filename}.`);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const parsedEntries = rawEntries.map(([name, filename, bytes]) => [
|
||||
name,
|
||||
parseJsonBytes(bytes, filename),
|
||||
]);
|
||||
parsedEntries.push(["shaManifest", shaManifest]);
|
||||
return validateAssetSet(Object.fromEntries(parsedEntries));
|
||||
}
|
||||
|
||||
export { SCHEMA_VERSION };
|
||||
14
dashboard/package.json
Normal file
14
dashboard/package.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "utah-vehicle-health-dashboard",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Dependency-free static dashboard for approved Utah Vehicle Health aggregates.",
|
||||
"scripts": {
|
||||
"start": "node server.mjs",
|
||||
"test": "node --test tests/contract.test.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
}
|
||||
1
dashboard/public/data/age_risk_curve.json
Normal file
1
dashboard/public/data/age_risk_curve.json
Normal file
@ -0,0 +1 @@
|
||||
{"development_preview":true,"population_estimate_allowed":false,"rows":[{"age_band":"0-3","nonpass_rate":0.058,"support_rounded":500},{"age_band":"4-7","nonpass_rate":0.06,"support_rounded":5900},{"age_band":"8-11","nonpass_rate":0.077,"support_rounded":9800},{"age_band":"12-15","nonpass_rate":0.109,"support_rounded":9700},{"age_band":"16-20","nonpass_rate":0.157,"support_rounded":8600},{"age_band":"21+","nonpass_rate":0.221,"support_rounded":4700}],"schema_version":"dashboard_data_v1"}
|
||||
1
dashboard/public/data/cohort_scorecard.json
Normal file
1
dashboard/public/data/cohort_scorecard.json
Normal file
@ -0,0 +1 @@
|
||||
{"development_preview":true,"population_estimate_allowed":false,"rows":[{"nonpass_rate":0.116,"prior_make":"FORD","prior_model":"F150","support_rounded":1100},{"nonpass_rate":0.09,"prior_make":"HONDA","prior_model":"ACCORD","support_rounded":400},{"nonpass_rate":0.073,"prior_make":"HONDA","prior_model":"CIVIC","support_rounded":300},{"nonpass_rate":0.086,"prior_make":"TOYOTA","prior_model":"CAMRY","support_rounded":500}],"schema_version":"dashboard_data_v1"}
|
||||
1
dashboard/public/data/coverage_quality.json
Normal file
1
dashboard/public/data/coverage_quality.json
Normal file
@ -0,0 +1 @@
|
||||
{"development_preview":true,"population_estimate_allowed":false,"rows":[{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3100,"utah_obd_proxy_share":0.0,"year":2016},{"labeled_rate":0.883,"overall_result_share":0.002,"source_era":"utah","support_rounded":600,"utah_obd_proxy_share":0.998,"year":2016},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":700,"utah_obd_proxy_share":0.0,"year":2016},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3200,"utah_obd_proxy_share":0.0,"year":2017},{"labeled_rate":0.899,"overall_result_share":0.0,"source_era":"utah","support_rounded":900,"utah_obd_proxy_share":1.0,"year":2017},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":700,"utah_obd_proxy_share":0.0,"year":2017},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3200,"utah_obd_proxy_share":0.0,"year":2018},{"labeled_rate":0.921,"overall_result_share":0.001,"source_era":"utah","support_rounded":1000,"utah_obd_proxy_share":0.999,"year":2018},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2018},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3300,"utah_obd_proxy_share":0.0,"year":2019},{"labeled_rate":0.93,"overall_result_share":0.001,"source_era":"utah","support_rounded":800,"utah_obd_proxy_share":0.999,"year":2019},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2019},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3200,"utah_obd_proxy_share":0.0,"year":2020},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2020},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":2200,"utah_obd_proxy_share":0.0,"year":2021},{"labeled_rate":0.931,"overall_result_share":0.0,"source_era":"utah","support_rounded":1100,"utah_obd_proxy_share":1.0,"year":2021},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2021},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3000,"utah_obd_proxy_share":0.0,"year":2022},{"labeled_rate":0.927,"overall_result_share":0.001,"source_era":"utah","support_rounded":1100,"utah_obd_proxy_share":0.999,"year":2022},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2022},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":3000,"utah_obd_proxy_share":0.0,"year":2023},{"labeled_rate":0.93,"overall_result_share":0.0,"source_era":"utah","support_rounded":1200,"utah_obd_proxy_share":1.0,"year":2023},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2023},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slc","support_rounded":500,"utah_obd_proxy_share":0.0,"year":2024},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"slco","support_rounded":300,"utah_obd_proxy_share":0.0,"year":2024},{"labeled_rate":0.929,"overall_result_share":0.0,"source_era":"utah","support_rounded":1100,"utah_obd_proxy_share":1.0,"year":2024},{"labeled_rate":1.0,"overall_result_share":1.0,"source_era":"weber","support_rounded":800,"utah_obd_proxy_share":0.0,"year":2024}],"schema_version":"dashboard_data_v1"}
|
||||
1
dashboard/public/data/data_manifest.json
Normal file
1
dashboard/public/data/data_manifest.json
Normal file
@ -0,0 +1 @@
|
||||
{"assets":["age_risk_curve.json","cohort_scorecard.json","coverage_quality.json","filter_catalog.json","model_diagnostics.json","overview_period_county.json"],"data_scope":{"first_year":2016,"last_year":2024,"model_names":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"]},"definitions":{"episode_gap_days":30,"locked_test_metrics_published":false,"support_rounding":100,"suppression_min_distinct_nonpass_vehicles":10,"suppression_min_distinct_pass_vehicles":10,"suppression_min_distinct_vehicles":100,"suppression_min_nonpass":10,"suppression_min_pass":10,"suppression_min_support":100,"target":"first-attempt next-episode binary non-pass rate"},"development_preview":true,"model_versions":["baseline_v1","hist_gradient_boosting_v1"],"population_estimate_allowed":false,"release_id":"7c1af8d22d3841b84d2cf4cd5ba7bce6a43578c78bc3d32a25d200b4b1986704","schema_version":"dashboard_data_v1"}
|
||||
1
dashboard/public/data/filter_catalog.json
Normal file
1
dashboard/public/data/filter_catalog.json
Normal file
@ -0,0 +1 @@
|
||||
{"age_bands":["0-3","4-7","8-11","12-15","16-20","21+"],"development_preview":true,"models":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"],"periods":[{"quarters":[1,2,3,4],"year":2016},{"quarters":[1,2,3,4],"year":2017},{"quarters":[1,2,3,4],"year":2018},{"quarters":[1,2,3,4],"year":2019},{"quarters":[1,2,3,4],"year":2020},{"quarters":[1,2,3,4],"year":2021},{"quarters":[1,2,3,4],"year":2022},{"quarters":[1,2,3,4],"year":2023},{"quarters":[1,2,3,4],"year":2024}],"population_estimate_allowed":false,"prior_make_models":[{"prior_make":"FORD","prior_model":"F150"},{"prior_make":"HONDA","prior_model":"ACCORD"},{"prior_make":"HONDA","prior_model":"CIVIC"},{"prior_make":"TOYOTA","prior_model":"CAMRY"}],"public_counties":["salt_lake","utah","weber"],"schema_version":"dashboard_data_v1"}
|
||||
1
dashboard/public/data/model_diagnostics.json
Normal file
1
dashboard/public/data/model_diagnostics.json
Normal file
@ -0,0 +1 @@
|
||||
{"development_preview":true,"population_estimate_allowed":false,"rows":[{"average_precision":0.275,"brier":0.0928,"log_loss":0.3218,"model":"hist_gradient_boosting_platt","partition":"calibrate","roc_auc":0.7072,"top_10_capture":0.2744},{"average_precision":0.3966,"brier":0.0877,"log_loss":0.3011,"model":"hist_gradient_boosting_raw","partition":"train","roc_auc":0.7871,"top_10_capture":0.3658},{"average_precision":0.2743,"brier":0.1011,"log_loss":0.3431,"model":"hist_gradient_boosting_raw","partition":"tune","roc_auc":0.7147,"top_10_capture":0.2635},{"average_precision":0.275,"brier":0.0928,"log_loss":0.3218,"model":"hist_gradient_boosting_raw","partition":"calibrate","roc_auc":0.7072,"top_10_capture":0.2744},{"average_precision":0.2573,"brier":0.0936,"log_loss":0.3261,"model":"logistic_platt","partition":"calibrate","roc_auc":0.6905,"top_10_capture":0.2729},{"average_precision":0.2482,"brier":0.0963,"log_loss":0.3327,"model":"logistic_raw","partition":"train","roc_auc":0.6955,"top_10_capture":0.2593},{"average_precision":0.2821,"brier":0.1012,"log_loss":0.3448,"model":"logistic_raw","partition":"tune","roc_auc":0.7072,"top_10_capture":0.2527},{"average_precision":0.2573,"brier":0.0936,"log_loss":0.3261,"model":"logistic_raw","partition":"calibrate","roc_auc":0.6905,"top_10_capture":0.2729},{"average_precision":0.146,"brier":0.1695,"log_loss":5.8429,"model":"previous_episode_literal","partition":"train","roc_auc":0.5687,"top_10_capture":0.2128},{"average_precision":0.1658,"brier":0.1723,"log_loss":5.9436,"model":"previous_episode_literal","partition":"tune","roc_auc":0.5821,"top_10_capture":0.2291},{"average_precision":0.1476,"brier":0.1677,"log_loss":5.7909,"model":"previous_episode_literal","partition":"calibrate","roc_auc":0.5767,"top_10_capture":0.224},{"average_precision":0.116,"brier":0.1025,"log_loss":0.3588,"model":"training_prevalence","partition":"train","roc_auc":0.5,"top_10_capture":0.1},{"average_precision":0.1246,"brier":0.1092,"log_loss":0.3764,"model":"training_prevalence","partition":"tune","roc_auc":0.5,"top_10_capture":0.1},{"average_precision":0.1129,"brier":0.1002,"log_loss":0.3526,"model":"training_prevalence","partition":"calibrate","roc_auc":0.5,"top_10_capture":0.1}],"schema_version":"dashboard_data_v1"}
|
||||
1
dashboard/public/data/overview_period_county.json
Normal file
1
dashboard/public/data/overview_period_county.json
Normal file
File diff suppressed because one or more lines are too long
1
dashboard/public/data/sha256_manifest.json
Normal file
1
dashboard/public/data/sha256_manifest.json
Normal file
@ -0,0 +1 @@
|
||||
{"development_preview":true,"files":[{"name":"age_risk_curve.json","sha256":"5445d52365ad3494ba05f097e7f4f5e41fffeecfe50fbb2952d72044f496c925"},{"name":"cohort_scorecard.json","sha256":"2ae83d9d55aa1f2ccae6420c66350fb491405eedfa90374aa3df2782a4aa0bf1"},{"name":"coverage_quality.json","sha256":"35b7dbed50e1b8260aac18c269f5045010ea23649a649f984a4693f3f00a1e83"},{"name":"data_manifest.json","sha256":"b7196b04eb223683c928cfb5375ed618e1230fa03382bd8a889de24c589557a0"},{"name":"filter_catalog.json","sha256":"9e45d428cd38853002ebd0bda089eb46d0832cd1b7b6b05ac953bc8472b01eb5"},{"name":"model_diagnostics.json","sha256":"ebf99eb32436a5eabb7d754cd88109f2cc0981575dd28966757031ac7faafa57"},{"name":"overview_period_county.json","sha256":"fe2a4233563b47fa31fc87859b100983d3bfb574c5a7fbc7241b9cfb4661b038"}],"population_estimate_allowed":false,"schema_version":"dashboard_data_v1"}
|
||||
82
dashboard/server.mjs
Normal file
82
dashboard/server.mjs
Normal file
@ -0,0 +1,82 @@
|
||||
import { createReadStream, realpathSync, statSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = realpathSync(path.dirname(fileURLToPath(import.meta.url)));
|
||||
const HOST = process.env.HOST || "127.0.0.1";
|
||||
const PORT = Number.parseInt(process.env.PORT || "4173", 10);
|
||||
const TYPES = new Map([
|
||||
[".css", "text/css; charset=utf-8"],
|
||||
[".html", "text/html; charset=utf-8"],
|
||||
[".js", "text/javascript; charset=utf-8"],
|
||||
[".json", "application/json; charset=utf-8"],
|
||||
[".mjs", "text/javascript; charset=utf-8"],
|
||||
[".svg", "image/svg+xml"],
|
||||
]);
|
||||
const PUBLIC_PATHS = new Set([
|
||||
"/index.html",
|
||||
"/styles.css",
|
||||
"/js/app.js",
|
||||
"/js/charts.js",
|
||||
"/js/data.js",
|
||||
"/public/data/age_risk_curve.json",
|
||||
"/public/data/cohort_scorecard.json",
|
||||
"/public/data/coverage_quality.json",
|
||||
"/public/data/data_manifest.json",
|
||||
"/public/data/filter_catalog.json",
|
||||
"/public/data/model_diagnostics.json",
|
||||
"/public/data/overview_period_county.json",
|
||||
"/public/data/sha256_manifest.json",
|
||||
]);
|
||||
|
||||
function safeFile(requestUrl) {
|
||||
let pathname;
|
||||
try {
|
||||
pathname = decodeURIComponent(new URL(requestUrl, "http://local").pathname);
|
||||
} catch {
|
||||
return { file: null, status: 400 };
|
||||
}
|
||||
if (pathname === "/") pathname = "/index.html";
|
||||
if (!PUBLIC_PATHS.has(pathname)) return { file: null, status: 404 };
|
||||
try {
|
||||
const candidate = realpathSync(path.join(ROOT, pathname.slice(1)));
|
||||
if (!candidate.startsWith(`${ROOT}${path.sep}`) || !statSync(candidate).isFile()) {
|
||||
return { file: null, status: 404 };
|
||||
}
|
||||
return { file: candidate, status: 200 };
|
||||
} catch {
|
||||
return { file: null, status: 404 };
|
||||
}
|
||||
}
|
||||
|
||||
const server = createServer((request, response) => {
|
||||
if (!request.url || !["GET", "HEAD"].includes(request.method || "")) {
|
||||
response.writeHead(405, { Allow: "GET, HEAD" });
|
||||
response.end("Method not allowed");
|
||||
return;
|
||||
}
|
||||
const resolved = safeFile(request.url);
|
||||
if (!resolved.file) {
|
||||
response.writeHead(resolved.status, { "Content-Type": "text/plain; charset=utf-8" });
|
||||
response.end(resolved.status === 400 ? "Bad request" : "Not found");
|
||||
return;
|
||||
}
|
||||
const file = resolved.file;
|
||||
const extension = path.extname(file).toLowerCase();
|
||||
response.writeHead(200, {
|
||||
"Content-Type": TYPES.get(extension) || "application/octet-stream",
|
||||
"Cache-Control": extension === ".json" ? "no-store" : "public, max-age=300",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Referrer-Policy": "no-referrer",
|
||||
});
|
||||
if (request.method === "HEAD") {
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
createReadStream(file).on("error", () => response.destroy()).pipe(response);
|
||||
});
|
||||
|
||||
server.listen(PORT, HOST, () => {
|
||||
process.stdout.write(`Utah Vehicle Health dashboard: http://${HOST}:${PORT}\n`);
|
||||
});
|
||||
1358
dashboard/styles.css
Normal file
1358
dashboard/styles.css
Normal file
File diff suppressed because it is too large
Load Diff
196
dashboard/tests/contract.test.mjs
Normal file
196
dashboard/tests/contract.test.mjs
Normal file
@ -0,0 +1,196 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash, webcrypto } from "node:crypto";
|
||||
import { readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
DataContractError,
|
||||
REQUIRED_ASSETS,
|
||||
SCHEMA_VERSION,
|
||||
loadDashboardData,
|
||||
validateAssetSet,
|
||||
} from "../js/data.js";
|
||||
|
||||
const DASHBOARD_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const PUBLIC_DATA = path.join(DASHBOARD_ROOT, "public", "data");
|
||||
|
||||
function json(filename) {
|
||||
return JSON.parse(readFileSync(path.join(PUBLIC_DATA, filename), "utf8"));
|
||||
}
|
||||
|
||||
function assetSet() {
|
||||
return Object.fromEntries(
|
||||
Object.entries(REQUIRED_ASSETS).map(([name, filename]) => [name, json(filename)]),
|
||||
);
|
||||
}
|
||||
|
||||
function publicDataFetch(overrides = new Map()) {
|
||||
return async (requestUrl) => {
|
||||
const filename = path.basename(new URL(String(requestUrl)).pathname);
|
||||
const source = path.join(PUBLIC_DATA, filename);
|
||||
let bytes;
|
||||
try {
|
||||
bytes = overrides.has(filename) ? overrides.get(filename) : readFileSync(source);
|
||||
} catch {
|
||||
return { ok: false, async arrayBuffer() { return new ArrayBuffer(0); } };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
async arrayBuffer() {
|
||||
return Uint8Array.from(bytes).buffer;
|
||||
},
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function filesRecursively(directory) {
|
||||
return readdirSync(directory).flatMap((name) => {
|
||||
const target = path.join(directory, name);
|
||||
return statSync(target).isDirectory() ? filesRecursively(target) : [target];
|
||||
});
|
||||
}
|
||||
|
||||
test("static shell exposes four semantic navigable views", () => {
|
||||
const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8");
|
||||
assert.match(html, /<header\b/);
|
||||
assert.match(html, /<nav\b[^>]*aria-label="Dashboard views"/);
|
||||
assert.match(html, /<main\b/);
|
||||
assert.match(html, /<footer\b/);
|
||||
for (const route of ["overview", "reliability", "estimator", "methods"]) {
|
||||
assert.match(html, new RegExp(`data-route="${route}"`));
|
||||
assert.match(html, new RegExp(`data-view="${route}"`));
|
||||
}
|
||||
assert.match(html, /2025 development-sample holdout was evaluated once/i);
|
||||
assert.match(html, /<strong>One-time holdout<\/strong><small>Already evaluated<\/small>/);
|
||||
assert.doesNotMatch(html, /<strong>Locked test<\/strong>|<small>Final evaluation<\/small>/);
|
||||
});
|
||||
|
||||
test("estimator remains disabled and contains no identifying input", () => {
|
||||
const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8");
|
||||
const estimator = html.match(/<section class="view" id="view-estimator"[\s\S]*?<\/section>/)?.[0];
|
||||
assert.ok(estimator, "estimator section is present");
|
||||
assert.match(estimator, /<fieldset disabled>/);
|
||||
const controlAttributes = [...estimator.matchAll(/<(?:input|select|textarea)\b([^>]*)>/g)].map(
|
||||
(match) => match[1],
|
||||
);
|
||||
for (const attributes of controlAttributes) {
|
||||
assert.doesNotMatch(
|
||||
attributes,
|
||||
/(?:name|id)\s*=\s*["'][^"']*(?:vin|plate|address|station|free.?text|zip)[^"']*["']/i,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("all required generated assets satisfy the browser contract", () => {
|
||||
const validated = validateAssetSet(assetSet());
|
||||
assert.equal(validated.manifest.schema_version, SCHEMA_VERSION);
|
||||
assert.equal(validated.manifest.development_preview, true);
|
||||
assert.equal(validated.manifest.population_estimate_allowed, false);
|
||||
assert.equal(validated.manifest.definitions.locked_test_metrics_published, false);
|
||||
assert.match(validated.manifest.release_id, /^[0-9a-f]{64}$/);
|
||||
assert.ok(validated.manifest.model_versions.length > 0);
|
||||
assert.deepEqual(
|
||||
validated.filters.partitions,
|
||||
["train", "tune", "calibrate"],
|
||||
);
|
||||
assert.ok(validated.overview.rows.length > 0);
|
||||
});
|
||||
|
||||
test("browser loader verifies every raw asset digest before rendering", async () => {
|
||||
const validated = await loadDashboardData({
|
||||
basePath: "http://dashboard.test/public/data/",
|
||||
fetchImplementation: publicDataFetch(),
|
||||
cryptoImplementation: webcrypto,
|
||||
});
|
||||
assert.equal(validated.manifest.schema_version, SCHEMA_VERSION);
|
||||
|
||||
const changedOverview = Buffer.concat([
|
||||
readFileSync(path.join(PUBLIC_DATA, REQUIRED_ASSETS.overview)),
|
||||
Buffer.from("\n"),
|
||||
]);
|
||||
await assert.rejects(
|
||||
loadDashboardData({
|
||||
basePath: "http://dashboard.test/public/data/",
|
||||
fetchImplementation: publicDataFetch(
|
||||
new Map([[REQUIRED_ASSETS.overview, changedOverview]]),
|
||||
),
|
||||
cryptoImplementation: webcrypto,
|
||||
}),
|
||||
(error) =>
|
||||
error instanceof DataContractError &&
|
||||
/Integrity verification failed for overview_period_county\.json/.test(error.message),
|
||||
);
|
||||
});
|
||||
|
||||
test("sha256 manifest covers and matches every approved data asset", () => {
|
||||
const manifest = json(REQUIRED_ASSETS.shaManifest);
|
||||
const expectedNames = Object.entries(REQUIRED_ASSETS)
|
||||
.filter(([name]) => name !== "shaManifest")
|
||||
.map(([, filename]) => filename)
|
||||
.sort();
|
||||
assert.deepEqual(
|
||||
manifest.files.map((file) => file.name).sort(),
|
||||
expectedNames,
|
||||
);
|
||||
for (const entry of manifest.files) {
|
||||
const digest = createHash("sha256")
|
||||
.update(readFileSync(path.join(PUBLIC_DATA, entry.name)))
|
||||
.digest("hex");
|
||||
assert.equal(digest, entry.sha256, `${entry.name} digest`);
|
||||
}
|
||||
});
|
||||
|
||||
test("contract fails closed for missing, inconsistent, or sensitive data", () => {
|
||||
const missing = assetSet();
|
||||
delete missing.overview;
|
||||
assert.throws(() => validateAssetSet(missing), DataContractError);
|
||||
|
||||
const inconsistent = assetSet();
|
||||
inconsistent.ageRisk.population_estimate_allowed = true;
|
||||
assert.throws(() => validateAssetSet(inconsistent), DataContractError);
|
||||
|
||||
const sensitive = assetSet();
|
||||
sensitive.scorecards.rows[0].vehicle_token = "not-public";
|
||||
assert.throws(() => validateAssetSet(sensitive), DataContractError);
|
||||
|
||||
const extraRowField = assetSet();
|
||||
extraRowField.overview.rows[0].note = "unapproved";
|
||||
assert.throws(() => validateAssetSet(extraRowField), DataContractError);
|
||||
|
||||
const lockedMetrics = assetSet();
|
||||
lockedMetrics.manifest.definitions.locked_test_metrics_published = true;
|
||||
assert.throws(() => validateAssetSet(lockedMetrics), DataContractError);
|
||||
|
||||
const extraPartition = assetSet();
|
||||
extraPartition.filters.partitions.push("locked_test");
|
||||
assert.throws(() => validateAssetSet(extraPartition), DataContractError);
|
||||
|
||||
const unsortedVersions = assetSet();
|
||||
unsortedVersions.manifest.model_versions = ["z_v1", "a_v1"];
|
||||
assert.throws(() => validateAssetSet(unsortedVersions), DataContractError);
|
||||
|
||||
const belowSuppression = assetSet();
|
||||
belowSuppression.overview.rows[0].support_rounded =
|
||||
belowSuppression.manifest.definitions.suppression_min_support - 1;
|
||||
assert.throws(() => validateAssetSet(belowSuppression), DataContractError);
|
||||
|
||||
const incorrectlyRounded = assetSet();
|
||||
incorrectlyRounded.coverage.rows[0].support_rounded += 1;
|
||||
assert.throws(() => validateAssetSet(incorrectlyRounded), DataContractError);
|
||||
});
|
||||
|
||||
test("dashboard source has no external runtime dependency or credential marker", () => {
|
||||
const sourceFiles = filesRecursively(DASHBOARD_ROOT).filter(
|
||||
(filename) =>
|
||||
!filename.includes(`${path.sep}public${path.sep}data${path.sep}`) &&
|
||||
!filename.includes(`${path.sep}tests${path.sep}`) &&
|
||||
/\.(?:html|css|js|mjs)$/.test(filename),
|
||||
);
|
||||
for (const filename of sourceFiles) {
|
||||
const source = readFileSync(filename, "utf8");
|
||||
assert.doesNotMatch(source, /(?:postgres(?:ql)?:\/\/|PGPASSWORD|PGHOST|countydata\.)/i);
|
||||
assert.doesNotMatch(source, /<script[^>]+src=["']https?:|@import\s+url\(["']?https?:/i);
|
||||
}
|
||||
});
|
||||
97
docs/dashboard_spec.md
Normal file
97
docs/dashboard_spec.md
Normal file
@ -0,0 +1,97 @@
|
||||
# Bolt dashboard specification
|
||||
|
||||
## Product language
|
||||
|
||||
Use **Utah Vehicle Health** as the brand, but call the modeled quantity
|
||||
**next-episode non-pass risk**. The estimator is a cohort estimate, not a
|
||||
diagnosis, certification, or guarantee.
|
||||
|
||||
## Four-page MVP
|
||||
|
||||
The sections below describe the full product target. The checked-in
|
||||
development preview intentionally implements a narrower safe subset: binary
|
||||
pass/non-pass aggregates, supported make/model scorecards, age bands, coverage
|
||||
quality, and pre-2025 model diagnostics. Four-class charts, uncertainty
|
||||
intervals, adjusted scorecards, and the prediction lookup remain disabled until
|
||||
their own reviewed aggregate assets exist.
|
||||
|
||||
### Overview
|
||||
|
||||
- Eligible inspections, pass rate, non-pass rate, and covered-period KPIs
|
||||
- Quarterly pass/fail/reject/abort trend with blanks shown separately
|
||||
- Covered-county map; unavailable counties remain gray
|
||||
- Non-pass risk versus vehicle age with intervals
|
||||
- Clear notices for partial periods and limited feed coverage
|
||||
|
||||
### Reliability explorer
|
||||
|
||||
- Search supported canonical make/model cohorts
|
||||
- Compare up to three cohorts across vehicle-age bands
|
||||
- Observed versus model-adjusted risk toggle
|
||||
- Outcome-mix bar and uncertainty-aware ranked dot plot
|
||||
- County, make/model, age band, fuel, program, and period filters
|
||||
- Support size and interval displayed for every estimate
|
||||
|
||||
### Next-test risk estimator
|
||||
|
||||
Inputs are coarsened, non-identifying attributes: county, supported make/model,
|
||||
vehicle-age band, fuel, prior episode outcome, time-since-prior band, season,
|
||||
and approved program category.
|
||||
|
||||
Output a calibrated non-pass probability, uncertainty interval, relevant
|
||||
baseline, and aggregate factor contributions. Never request VIN, plate, exact
|
||||
address, station, free text, or current-test diagnostics.
|
||||
|
||||
### Data and methods
|
||||
|
||||
- Coverage timeline and source/year missingness heatmap
|
||||
- Episode and target definitions
|
||||
- `slc`/`slco` source-era explanation
|
||||
- Temporal split, PR-AUC, Brier score, and calibration plot
|
||||
- Subgroup/source-era performance
|
||||
- Leakage controls, DMV gaps, partial periods, and limitations
|
||||
|
||||
## Public data contract
|
||||
|
||||
| Dataset | Safe grain |
|
||||
| --- | --- |
|
||||
| `data_manifest` | Data cutoff, deterministic release ID, model versions, definitions and exclusions |
|
||||
| `overview_period_county` | Quarter/year × public county with rounded support and outcome rates |
|
||||
| `cohort_scorecard` | Approved make/model × age band, optionally coarsened county/fuel |
|
||||
| `age_risk_curve` | Approved cohort × age point/band with risk, interval and support |
|
||||
| `prediction_lookup` | Only supported coarsened input combinations and calibrated outputs |
|
||||
| `filter_catalog` | Publishable categories and valid combinations |
|
||||
| `model_diagnostics` | Approved partition-level metrics; locked metrics require a separate release gate |
|
||||
| `coverage_quality` | Source era × year volume, blank rate, linkage and availability |
|
||||
|
||||
Use purpose-built outputs rather than a single high-dimensional browser cube.
|
||||
Bolt receives only these sanitized, versioned assets—never countydata
|
||||
credentials or private analytical rows.
|
||||
|
||||
## Publication controls
|
||||
|
||||
- Suppress cells below 100 eligible inspections.
|
||||
- Also require at least 100 distinct private vehicle tokens in every published
|
||||
cell; tokens and distinct counts never enter the public asset.
|
||||
- Suppress when an outcome or its complement has fewer than 10 records.
|
||||
- Require at least 10 distinct vehicles contributing each binary class.
|
||||
- Apply complementary suppression so totals cannot reconstruct hidden cells.
|
||||
- Combine rare categories, coarsen model years, and round displayed counts.
|
||||
- Recheck thresholds after every filter combination.
|
||||
- Do not include suppressed rows in browser bundles, API responses, downloads,
|
||||
analytics logs, or hidden chart layers.
|
||||
- Downloads contain only the sanitized summary currently displayed.
|
||||
|
||||
## Visual direction
|
||||
|
||||
Use a restrained Utah/desert palette: teal pass, red fail, amber reject, purple
|
||||
abort, and gray missing. Use probability bars, calibrated dot plots, confidence
|
||||
bands, and cohort comparisons instead of gauges. Do not rely on color alone.
|
||||
|
||||
## Stretch pages
|
||||
|
||||
- Failure-to-pass journeys with funnels, attempts-to-pass and survival curves
|
||||
- Four-class outcome probabilities
|
||||
- Source-scoped OBD early-warning analysis
|
||||
- Automated aggregate refresh with a dedicated read-only role
|
||||
- Model-drift monitoring
|
||||
102
docs/data_inventory.md
Normal file
102
docs/data_inventory.md
Normal file
@ -0,0 +1,102 @@
|
||||
# Countydata inventory
|
||||
|
||||
Inventory date: 2026-07-15. All inspection was performed with read-only
|
||||
transactions and aggregate queries; no identifiers were exported.
|
||||
|
||||
## Accessible databases
|
||||
|
||||
- `countydata`: the useful analytical database, approximately 93.4 GB.
|
||||
- `postgres`: empty local/staging copies of the core vehicle table structure
|
||||
plus foreign-table links. It is not the analytical source.
|
||||
- `vattp`: a tiny, unrelated course/event registration and survey application.
|
||||
It is not suitable for this project and its application records should remain
|
||||
out of scope.
|
||||
|
||||
## DMV logical dataset
|
||||
|
||||
The following are one logical dataset split into projections joined one-to-one
|
||||
by `id`; they are not four independent populations.
|
||||
|
||||
| Relation | Approximate role | Key analytical fields |
|
||||
| --- | --- | --- |
|
||||
| `data.dmv_tax_json` | Raw JSON and ingest metadata | Full DMV record, file/source metadata |
|
||||
| `data.dmv_tax` | Core lookup | VIN, registration date, county |
|
||||
| `data.dmv_tax_search` | Search projection | VIN, registration date, county |
|
||||
| `data.dmv_tax_vehicle` | Vehicle projection | Make, model, model year, fuel, registration type/place, temporary flag, expiration/emission dates, ZIPs |
|
||||
|
||||
Exact current logical row count: **18,009,278**.
|
||||
|
||||
- Date bounds are 2011-09-07 through 2024-03-13, but the few pre-2016 rows are
|
||||
outliers and there are only 117 records in 2021.
|
||||
- The 29 Utah counties are represented. Salt Lake (35.66%), Utah (17.21%),
|
||||
Davis (10.29%), and Weber (8.01%) account for most records.
|
||||
- Canonicalized fuel mix by record is 86.23% gasoline, 7.97% diesel, 2.59%
|
||||
flexible fuel, 2.06% hybrid, 0.70% electric, and 0.24% plug-in hybrid.
|
||||
- A vehicle appears repeatedly over time: sampled DMV histories had a median of
|
||||
five registration records.
|
||||
|
||||
## Inspection logical dataset
|
||||
|
||||
These are likewise projections of one logical inspection dataset joined by
|
||||
`id`.
|
||||
|
||||
| Relation | Approximate role | Key analytical fields |
|
||||
| --- | --- | --- |
|
||||
| `data.inspection_json` | Raw JSON and ingest metadata | Full inspection, detailed OBD/readiness/visual fields where supplied |
|
||||
| `data.inspection` | Core lookup | VIN, test timestamp, ingest timestamp |
|
||||
| `data.inspection_search` | Outcome/search projection | County/source, overall and OBD results, test/program type |
|
||||
| `data.inspection_vehicle` | Vehicle projection | Make, model, year, calibration/certificate, station |
|
||||
| `data.inspection_obd` | OBD summary | Result-reason code, DTC count, permanent-DTC flag |
|
||||
| `data.inspection_plate` | Identifier lookup | VIN, plate, test timestamp; sensitive and unnecessary for analytics |
|
||||
|
||||
Exact current logical row count: **19,357,287**.
|
||||
|
||||
- Four dates in 1990 are outliers. Normal coverage begins in 2010 and continues
|
||||
through 2026-06-22; 2026 is partial.
|
||||
- Real source/county labels are `slc`, `slco`, `utah`, `weber`, `davis`, and
|
||||
`cache`. The two Salt Lake labels represent different source eras and should
|
||||
not be blindly treated as different counties.
|
||||
- Overall results are 71.06% pass, 3.61% fail, 3.56% reject, 2.31% abort, and
|
||||
19.45% blank/null or other near-blank values in the raw `overall_result`
|
||||
field. Most missing overall results belong to the older Utah County feed; its
|
||||
audited OBD/OBD rows carry a separate result that is usable only as a
|
||||
provenance-tagged binary pass-versus-non-pass proxy.
|
||||
- About 89.5% of records use the OBD program and about 9.2% use TSI.
|
||||
- DTC count is zero in about 90.4% of records and null in about 1.4%; the
|
||||
remaining values are class-imbalanced and include rare data-quality outliers.
|
||||
- Raw JSON can contain odometer, vehicle fuel/type/GVWR/cylinders/engine,
|
||||
transmission, DTCs, PIDs, MIL/readiness status, communication protocol, and
|
||||
visual inspection fields. These richer fields are concentrated in the newer
|
||||
`slco`, `davis`, and `cache` feeds rather than statewide history.
|
||||
|
||||
## Longitudinal linkage
|
||||
|
||||
VIN is indexed in both logical datasets and makes longitudinal analysis
|
||||
possible, but must never appear in public outputs.
|
||||
|
||||
- 88.2% of 10,000 sampled distinct inspection VINs had at least one DMV match.
|
||||
- The sampled inspection history median was eight visits per vehicle.
|
||||
- 74.2% of 10,000 sampled distinct DMV VINs had at least one inspection match.
|
||||
- Extreme repeat counts exist and require invalid/shared-identifier filtering.
|
||||
|
||||
Use a salted one-way internal token if a stable identifier is needed during
|
||||
feature engineering. Never send VINs or plates to the browser or Bolt.
|
||||
|
||||
## Operational and sensitive relations
|
||||
|
||||
The `imreports` and `fdw_data` schemas include users, invitations, sessions,
|
||||
event/upload logs, file contents, client network metadata, and remote ingest
|
||||
state. They are operational rather than analytical and should be excluded.
|
||||
Foreign tables in `fdw_countydata` mirror source data and are unnecessary when
|
||||
the normalized `data` tables are available.
|
||||
|
||||
## Primary quality risks
|
||||
|
||||
- Missing years and partial periods can masquerade as real trends.
|
||||
- Make/model and fuel categories need canonicalization.
|
||||
- Outcome blanks must not be treated as passes.
|
||||
- County labels also encode source-system changes and therefore potential drift.
|
||||
- Rich JSON features are missing by design in older feeds, not missing at
|
||||
random.
|
||||
- Same-test OBD/result fields cause target leakage in pre-test prediction.
|
||||
- Direct identifiers and small groups require aggregation and suppression.
|
||||
136
docs/development_results.md
Normal file
136
docs/development_results.md
Normal file
@ -0,0 +1,136 @@
|
||||
# Development run results
|
||||
|
||||
Run date: 2026-07-15
|
||||
|
||||
These results validate the engineering and modeling pipeline. They are **not
|
||||
population estimates**: the development extract samples inspection-table pages
|
||||
and then retrieves complete histories for up to 10,000 vehicles, which
|
||||
over-represents vehicles with more inspection records.
|
||||
|
||||
## Label audit
|
||||
|
||||
The raw Utah County feed leaves `overall_result` blank for 3,747,862 records.
|
||||
An aggregate-only server audit found 3,469,102 Utah `obd`/`OBD` records with a
|
||||
controlled pass, fail, reject, or abort value in `obd_result`. Across non-Utah
|
||||
feeds where both fields were recognized, the two fields agreed on binary pass
|
||||
versus non-pass 99.30% of the time, but their fail-versus-reject categories were
|
||||
not interchangeable.
|
||||
|
||||
Label contract v3 therefore permits the Utah OBD value only as a provenance-
|
||||
tagged binary proxy. TSI, `other/C`, `B`, blank, and unknown values stay
|
||||
unlabeled. Four-class analysis must use `overall_result` exclusively. This also
|
||||
matches the official program distinction between a readiness rejection and a
|
||||
failed inspection; see the [Utah inspection requirements](https://dmv.utah.gov/register/inspections/)
|
||||
and [program definition of rejection](https://www.utah.gov/pmn/files/1155003.pdf).
|
||||
|
||||
In the development extract, the rule restored 11,334 labels and left 1,281
|
||||
events unlabeled. The label-source field remains private audit metadata and is
|
||||
not a model feature.
|
||||
|
||||
## Pipeline reconciliation
|
||||
|
||||
| Stage | Rows |
|
||||
| --- | ---: |
|
||||
| Source events written after VIN validation | 83,552 |
|
||||
| Clean events after duplicate/conflict handling | 80,190 |
|
||||
| Inspection episodes | 69,588 |
|
||||
| Eligible returning targets | 44,659 |
|
||||
|
||||
The sample contains 9,996 retained vehicle tokens. The extraction manifest,
|
||||
compressed-file SHA-256, mart manifest, Parquet SHA-256, and row counts all
|
||||
reconcile. Every mart invariant reports zero violations. Raw VIN and raw OBD
|
||||
result are absent from the staging output schema, feature mart, and model
|
||||
features.
|
||||
|
||||
| Temporal partition | Eligible targets | Never-fit audit targets |
|
||||
| --- | ---: | ---: |
|
||||
| Train, 2016–2022 | 31,745 | 3,129 |
|
||||
| Tune, 2023 | 4,972 | 526 |
|
||||
| Calibrate, 2024 | 2,633 | 277 |
|
||||
| Locked test, 2025 | 4,451 | 480 |
|
||||
| Shadow, 2026 partial | 858 | 106 |
|
||||
|
||||
Audit-bucket vehicles are excluded from fitting, tuning, and calibration. The
|
||||
normal trainers do not calculate 2025 outcomes or metrics without the explicit
|
||||
locked-evaluation flag.
|
||||
|
||||
## Baseline results before the locked test
|
||||
|
||||
The table below uses only non-audit vehicles. Average precision is the project's
|
||||
PR-AUC summary. The calibrated logistic row is shown only on the partition used
|
||||
to fit the calibrator and is therefore a calibration diagnostic, not an
|
||||
independent final estimate.
|
||||
|
||||
| Partition | Model | PR-AUC | Brier | ROC-AUC | Precision at top 10% |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| 2023 tune | Training prevalence | 0.125 | 0.1092 | 0.500 | 0.125 |
|
||||
| 2023 tune | Previous episode, literal | 0.166 | 0.1723 | 0.582 | 0.285 |
|
||||
| 2023 tune | Logistic | **0.282** | **0.1012** | **0.707** | **0.315** |
|
||||
| 2024 calibrate | Training prevalence | 0.113 | 0.1002 | 0.500 | 0.113 |
|
||||
| 2024 calibrate | Previous episode, literal | 0.148 | 0.1677 | 0.577 | 0.253 |
|
||||
| 2024 calibrate | Logistic | 0.257 | 0.0936 | 0.690 | 0.308 |
|
||||
| 2024 calibrate | Logistic + Platt | **0.257** | **0.0936** | **0.690** | **0.308** |
|
||||
|
||||
The selected logistic regularization was `C=0.03`. It converged in 2,390 of the
|
||||
5,000 allowed iterations; Platt calibration converged in five iterations. All
|
||||
five candidates and all stored probabilities passed explicit convergence and
|
||||
finite-value checks.
|
||||
|
||||
The literal previous-outcome baseline is useful as a ranking sanity check but
|
||||
produces overconfident zero/one probabilities, explaining its poor Brier and log
|
||||
loss. The transparent logistic model is the current development leader: on the
|
||||
2023 tuning partition it more than doubles prevalence PR-AUC and raises top-10%
|
||||
precision from 12.5% to 31.5%.
|
||||
|
||||
## One-time development holdout
|
||||
|
||||
The nonlinear live-verification command invoked the explicit 2025 gate after
|
||||
its feature contract and four-candidate grid had already been fixed and its
|
||||
candidate had been selected only on 2023. No test-driven model change was made.
|
||||
At that point the development specification was frozen and the already-fixed
|
||||
logistic model was evaluated once for a direct comparison. These are
|
||||
development-sample holdout results, not final population claims.
|
||||
|
||||
| 2025 non-audit model | PR-AUC | Brier | ROC-AUC | Precision at top 10% |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Training prevalence | 0.123 | 0.1076 | 0.500 | 0.123 |
|
||||
| Previous episode, literal | 0.158 | 0.1731 | 0.576 | 0.268 |
|
||||
| Logistic + Platt | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| Histogram gradient boosting + Platt | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
|
||||
The holdout contains 3,971 non-audit episodes from 3,734 vehicles. The
|
||||
never-fit audit contains another 480 episodes from 450 vehicles; calibrated
|
||||
logistic PR-AUC is 0.253 and Brier is 0.0859 there. The logistic model beats
|
||||
both simple baselines on the project's headline metrics and remains the chosen
|
||||
development model. The tree did not provide a decisive pre-test improvement
|
||||
that justified its added complexity, and no further 2025-informed tuning is
|
||||
permitted.
|
||||
|
||||
## Remaining gates
|
||||
|
||||
1. Run a complete, contiguous bounded extraction for publishable population
|
||||
aggregates; the page-sampled development cohort cannot support dashboard
|
||||
prevalence or county rankings.
|
||||
2. Treat any later full-data 2025 result as confirmatory rather than a pristine
|
||||
unseen test, because the development sample's holdout has now been opened.
|
||||
3. Run the frozen subgroup/source-era report and clustered uncertainty
|
||||
analysis on the complete extraction.
|
||||
4. Replace the checked-in development-preview bundle with complete-data
|
||||
aggregates only after the population-publication review passes.
|
||||
|
||||
## Sanitized dashboard preview
|
||||
|
||||
The repository now includes a fail-closed exporter and a static four-view
|
||||
dashboard shell. The preview exporter reads only train, tune, and calibration
|
||||
rows dated before 2025. It rejects any model manifest that says the holdout was
|
||||
evaluated, rejects any metric row outside the three approved partitions, and
|
||||
marks every asset `development_preview=true` and
|
||||
`population_estimate_allowed=false`.
|
||||
|
||||
Published cells must clear minimum episode and distinct-vehicle thresholds,
|
||||
including both binary-class complements. Supports are rounded, direct and
|
||||
pseudonymous identifiers stay private, and a checksum manifest covers the
|
||||
approved JSON bundle. Private model manifests bind the exact metrics files, and
|
||||
the public manifest exposes a deterministic release ID plus model versions for
|
||||
provenance. The site disables the estimator because no privacy-reviewed
|
||||
prediction lookup exists.
|
||||
197
docs/model_card.md
Normal file
197
docs/model_card.md
Normal file
@ -0,0 +1,197 @@
|
||||
# Utah Vehicle Health model card
|
||||
|
||||
Last updated: 2026-07-15
|
||||
|
||||
Status: **development prototype; not approved for production or population claims**
|
||||
|
||||
## Model summary
|
||||
|
||||
Utah Vehicle Health estimates the probability that the first attempt of a
|
||||
returning vehicle's next emissions-inspection episode will be a non-pass. The
|
||||
current selected model is a regularized logistic regression followed by Platt
|
||||
probability calibration. It is an inspection-history model, not a mechanical
|
||||
health, safety, roadworthiness, or legal-compliance model.
|
||||
|
||||
This model card summarizes the current page-sampled development run. The full
|
||||
research and product contracts are in the [project charter](project_charter.md),
|
||||
[modeling protocol](modeling_protocol.md), and
|
||||
[dashboard specification](dashboard_spec.md).
|
||||
|
||||
## Intended use and prohibited use
|
||||
|
||||
Intended uses are to validate the data-engineering and modeling pipeline,
|
||||
compare leakage-safe model candidates, study aggregate patterns, and support a
|
||||
clearly marked development dashboard preview.
|
||||
|
||||
Do not use the model to:
|
||||
|
||||
- make decisions about an individual vehicle, owner, registration, inspection,
|
||||
or station;
|
||||
- diagnose a vehicle, guarantee an inspection result, or infer safety or
|
||||
roadworthiness;
|
||||
- rank or penalize people, counties, programs, or inspection stations;
|
||||
- make causal claims from observed associations; or
|
||||
- report statewide prevalence, county rankings, or production performance from
|
||||
the current development sample.
|
||||
|
||||
The public estimator remains disabled because no privacy-reviewed prediction
|
||||
lookup has been approved.
|
||||
|
||||
## Prediction unit and target
|
||||
|
||||
The prediction is made immediately before a new inspection episode begins. An
|
||||
episode groups consecutive attempts no more than 30 days apart, and only its
|
||||
first attempt is the supervised target. A vehicle must have at least one prior
|
||||
completed episode to enter the returning-vehicle cohort.
|
||||
|
||||
The binary target is:
|
||||
|
||||
- `0`: recognized pass;
|
||||
- `1`: recognized fail, reject, or abort; and
|
||||
- unlabeled: blank, null, or unrecognized results.
|
||||
|
||||
A recognized overall result is preferred. For the older Utah County OBD feed,
|
||||
a controlled OBD result may fill a blank overall result only under the narrow
|
||||
source/program/test contract documented in the
|
||||
[modeling protocol](modeling_protocol.md#source-specific-label-contract). That
|
||||
proxy is approved only for binary pass versus non-pass; it must not support a
|
||||
four-class interpretation. Label provenance is private audit metadata and is
|
||||
not a predictor.
|
||||
|
||||
## Development data and representativeness
|
||||
|
||||
The current extract samples inspection-table pages and then retrieves complete
|
||||
histories for up to 10,000 vehicles. This over-represents vehicles with more
|
||||
inspection records. The run contains 83,552 retained source events, 69,588
|
||||
episodes, and 44,659 eligible returning targets, but those counts do not make
|
||||
the sample population-representative.
|
||||
|
||||
Coverage is limited to participating county/source feeds and changes over time.
|
||||
It does not represent all 29 Utah counties. Source-system transitions, missing
|
||||
years, and partial periods can resemble real changes in risk. See the
|
||||
[data inventory](data_inventory.md) and
|
||||
[development results](development_results.md) for the audited scope.
|
||||
|
||||
## Features and leakage exclusions
|
||||
|
||||
The selected model uses information available before the target episode:
|
||||
|
||||
- vehicle age;
|
||||
- prior episode and attempt counts;
|
||||
- days since the prior episode and prior adverse outcome;
|
||||
- prior non-pass rate;
|
||||
- prior first and final outcomes;
|
||||
- previously observed make and model;
|
||||
- public county; and
|
||||
- target season.
|
||||
|
||||
Categorical mappings and all preprocessing are fit on training data only. The
|
||||
current model does not yet include DMV enrichment or rich same-test OBD fields.
|
||||
|
||||
Excluded inputs include direct or pseudonymous identifiers, station
|
||||
information, the target attempt's result or diagnostics, later attempts in the
|
||||
target episode, future records, full-history aggregates, label provenance, and
|
||||
preprocessing learned from evaluation periods. The complete exclusion contract
|
||||
is in [modeling_protocol.md](modeling_protocol.md#leakage-exclusions).
|
||||
|
||||
## Chronology and holdout status
|
||||
|
||||
The fixed temporal design is:
|
||||
|
||||
| Period | Role |
|
||||
| --- | --- |
|
||||
| 2010-2015 | Historical context only |
|
||||
| 2016-2022 | Fit preprocessing and models |
|
||||
| 2023 | Select hyperparameters and compare candidates |
|
||||
| 2024 | Fit probability calibration and inspect calibration behavior |
|
||||
| 2025 | One-time development holdout |
|
||||
| 2026 partial | Shadow monitoring only |
|
||||
|
||||
During live verification, the explicit 2025 gate was opened once after both
|
||||
candidate specifications and the tree search grid had already been fixed from
|
||||
pre-2025 data. No model was changed in response. The already-fixed logistic
|
||||
model was then evaluated once for a direct comparison. Consequently, this
|
||||
page-sampled 2025 cohort is no longer a pristine unseen test, and no further
|
||||
2025-informed tuning is permitted. Any complete-data 2025 analysis must be
|
||||
described as confirmatory. The detailed audit trail is in
|
||||
[development_results.md](development_results.md#one-time-development-holdout).
|
||||
|
||||
## Candidate comparison and selected model
|
||||
|
||||
The candidates were training prevalence, the literal previous-episode outcome,
|
||||
regularized logistic regression, and histogram gradient boosting. Results below
|
||||
use non-audit development rows.
|
||||
|
||||
| Evaluation | Model | PR-AUC | Brier | ROC-AUC | Top-10% precision |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| 2023 tuning | Training prevalence | 0.125 | 0.1092 | 0.500 | 0.125 |
|
||||
| 2023 tuning | Previous episode | 0.166 | 0.1723 | 0.582 | 0.285 |
|
||||
| 2023 tuning | Logistic | **0.282** | **0.1012** | **0.707** | **0.315** |
|
||||
| 2025 one-time holdout | Logistic + Platt | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| 2025 one-time holdout | Gradient boosting + Platt | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
|
||||
The selected logistic model uses `C=0.03`. It satisfied explicit convergence
|
||||
and finite-value checks. Platt scaling was fit on the 2024 calibration cohort;
|
||||
on that same cohort, PR-AUC was 0.257 and Brier score was 0.0936. Those 2024
|
||||
values are calibration diagnostics, not independent final performance.
|
||||
|
||||
The logistic model remains selected because it beat both simple baselines and
|
||||
the more complex tree on the one-time development holdout while remaining more
|
||||
transparent. The separate never-fit vehicle audit also supported the logistic
|
||||
model, but the development sample is too limited for final generalization or
|
||||
fairness claims.
|
||||
|
||||
## Coverage, fairness, and privacy limitations
|
||||
|
||||
- The model applies only to returning vehicles with recognizable labels and
|
||||
sufficient prior history; cold-start behavior is not established.
|
||||
- Geography and source era are entangled. Performance may shift when a feed,
|
||||
county program, vehicle mix, or label process changes.
|
||||
- Reject and abort outcomes can reflect readiness or process issues rather than
|
||||
mechanical failure.
|
||||
- No protected attributes are modeled, but their absence does not establish
|
||||
fairness. Subgroup sample sizes, errors, and calibration still require review.
|
||||
- Make/model normalization and complete-data subgroup analysis are unfinished.
|
||||
- The current probability calibration has not yet received independent,
|
||||
vehicle-clustered uncertainty analysis or external validation.
|
||||
|
||||
Private analytical artifacts remain local. Public assets contain only reviewed,
|
||||
rounded aggregates that satisfy episode, distinct-vehicle, and binary-class
|
||||
suppression thresholds. Direct identifiers, private linkage values, raw rows,
|
||||
and row-level predictions are outside the public contract. See
|
||||
[dashboard_spec.md](dashboard_spec.md#publication-controls).
|
||||
|
||||
## Monitoring
|
||||
|
||||
Before any deployment, monitoring must cover source-level volume, label
|
||||
recognition, outcome prevalence, source/program mix, missingness, unseen
|
||||
categories, score distributions, PR-AUC, Brier score, and calibration. Reviews
|
||||
must explicitly separate the Salt Lake source transition, newer rich feeds, DMV
|
||||
coverage gaps, and the partial 2026 period. Alert thresholds and a response plan
|
||||
remain to be defined.
|
||||
|
||||
## Reproducibility
|
||||
|
||||
The pipeline uses fixed temporal boundaries and a fixed random seed. Python
|
||||
dependencies are pinned in [requirements.txt](../requirements.txt). Private
|
||||
manifests record input lineage, software/model configuration, convergence, row
|
||||
reconciliation, and checksums without publishing private paths or values. SQL
|
||||
transforms and validation queries are versioned under [sql](../sql), model
|
||||
runners are under [scripts](../scripts), and automated checks are under
|
||||
[tests](../tests). Normal training commands leave the 2025 evaluation gate
|
||||
closed unless an explicit flag is supplied.
|
||||
|
||||
## Remaining approval gates
|
||||
|
||||
1. Run a complete, contiguous, bounded extraction and rebuild the frozen
|
||||
pipeline without page-sampling bias.
|
||||
2. Treat complete-data 2025 results as confirmation and reserve a genuinely new
|
||||
period or external dataset for future unseen evaluation.
|
||||
3. Complete subgroup/source-era reporting, vehicle-clustered uncertainty,
|
||||
calibration diagnostics, and fairness review.
|
||||
4. Finish make/model normalization, episode-gap sensitivity checks, and the
|
||||
preregistered inspection-only versus DMV-enhanced ablation.
|
||||
5. Pass privacy and publication review before replacing the development-preview
|
||||
aggregates or enabling any prediction lookup.
|
||||
6. Define monitoring thresholds, ownership, rollback criteria, and a model
|
||||
update policy before operational use.
|
||||
141
docs/modeling_protocol.md
Normal file
141
docs/modeling_protocol.md
Normal file
@ -0,0 +1,141 @@
|
||||
# Leakage-safe modeling protocol
|
||||
|
||||
## Prediction unit
|
||||
|
||||
Score the **first attempt of the next inspection episode** immediately before
|
||||
check-in. Consecutive attempts for the same private vehicle token belong to the
|
||||
same episode when they are no more than 30 days apart. Re-run the analysis with
|
||||
14- and 45-day gaps as sensitivity checks.
|
||||
|
||||
This prevents rapid fail/retest sequences from dominating the target. Those
|
||||
within-episode attempts belong in the separate failure-to-pass journey analysis.
|
||||
|
||||
## Eligibility
|
||||
|
||||
- Use 2010-2015 events only as historical context; supervised targets begin in
|
||||
2016.
|
||||
- Exclude the four 1990 date outliers.
|
||||
- Require a recognized first-attempt outcome and at least one prior completed
|
||||
episode for the returning-vehicle model.
|
||||
- Keep first-observed vehicles as a separate cold-start cohort.
|
||||
- Normalize `slc` and `slco` to Salt Lake County for geography while preserving
|
||||
source era for drift reporting.
|
||||
- Deduplicate exact uploads and quarantine shared/test/placeholder identifiers
|
||||
using preregistered rules for impossible conflicts or extreme activity.
|
||||
- Do not require a DMV match; retain explicit match and staleness indicators.
|
||||
|
||||
Every exclusion must appear in a cohort-flow report.
|
||||
|
||||
## Source-specific label contract
|
||||
|
||||
The normalized target prefers a recognized `overall_result` for every source.
|
||||
The older Utah County feed is the sole exception: when its overall result is
|
||||
blank, `obd_result` may supply the binary target only when the source is
|
||||
`utah`, the program is `obd`, the test type is `OBD`, and the controlled value
|
||||
is pass, fail, reject, or abort. `B`, blank, TSI, `other/C`, and unknown values
|
||||
remain unlabeled. Every target retains `target_outcome_label_source` as private
|
||||
audit metadata, and that field is never a predictor.
|
||||
|
||||
This is a binary pass-versus-non-pass proxy, not a four-class substitution.
|
||||
Across non-Utah feeds where both fields are recognized, an aggregate audit found
|
||||
99.30% agreement on pass versus non-pass, while the fail/reject distinction was
|
||||
not interchangeable. Utah program rules also distinguish a readiness rejection
|
||||
from a failed inspection. Future four-class analysis must therefore require
|
||||
`target_outcome_label_source = 'overall_result'`. See the official
|
||||
[Utah inspection requirements](https://dmv.utah.gov/register/inspections/) and
|
||||
[program definition of rejection](https://www.utah.gov/pmn/files/1155003.pdf).
|
||||
|
||||
## Point-in-time features
|
||||
|
||||
Every historical window ends strictly before the target episode:
|
||||
|
||||
- prior episode and attempt counts;
|
||||
- previous episode first/final outcome;
|
||||
- expanding and trailing prior pass/fail/reject/abort counts and rates;
|
||||
- days since the previous episode and prior adverse result;
|
||||
- attempts required in prior episodes;
|
||||
- vehicle age and canonical make/model from prior information;
|
||||
- physical county and target month/season;
|
||||
- latest DMV record dated before the target, registration count/recency, fuel,
|
||||
and DMV match/staleness indicators; and
|
||||
- missingness indicators.
|
||||
|
||||
The stable core model is inspection-only. The DMV-enhanced model is an explicit
|
||||
ablation because DMV history ends in March 2024 and is nearly absent in 2021.
|
||||
|
||||
## Leakage exclusions
|
||||
|
||||
The MVP must not use:
|
||||
|
||||
- the target attempt's overall/OBD result, result reason, DTC count, MIL or
|
||||
readiness values, PIDs, visual checks, measurements, certificate, or
|
||||
calibration fields;
|
||||
- later attempts or eventual outcome from the target episode;
|
||||
- station or station-level outcome statistics;
|
||||
- raw VIN, private token, plate, ZIP, or other identifiers as features;
|
||||
- target-row vehicle attributes when a prior/static source is available;
|
||||
- future DMV records;
|
||||
- full-history aggregates; or
|
||||
- preprocessing, category mappings, target encodings, or imputation learned
|
||||
from validation/test data.
|
||||
|
||||
Timestamp ties must be resolved before lag/window calculations. Cumulative
|
||||
windows end at the preceding event or episode.
|
||||
|
||||
## Fixed evaluation timeline
|
||||
|
||||
| Partition | Target dates | Purpose |
|
||||
| --- | --- | --- |
|
||||
| Historical context | 2010-2015 | Lag features only |
|
||||
| Train | 2016-2022 | Fit preprocessing and models |
|
||||
| Tune | 2023 | Hyperparameters and selection |
|
||||
| Calibrate | 2024 | Probability calibration and thresholds |
|
||||
| Locked test | 2025 | Final reported performance |
|
||||
| Shadow drift | 2026-01-01 to 2026-06-22 | Monitoring only |
|
||||
|
||||
The page-sampled development extract's 2025 gate was opened once during live
|
||||
verification after both candidate specifications were fixed. It is therefore
|
||||
a one-time development holdout, not a pristine future test. No model change was
|
||||
made from it; a later complete-data 2025 run is confirmatory. See
|
||||
[development_results.md](development_results.md) for the audit trail. The table
|
||||
continues to define the frozen chronology for a complete extraction.
|
||||
|
||||
Repeated vehicles may cross ordinary time partitions because returning-vehicle
|
||||
prediction is the deployment scenario. Separately reserve 10% of keyed vehicle
|
||||
buckets as a never-fit VIN audit and report its 2025 performance as an unseen-
|
||||
vehicle stress test. Never use random row splitting.
|
||||
|
||||
## Baselines and candidate model
|
||||
|
||||
1. Training prevalence
|
||||
2. Repeat the previous episode's first outcome
|
||||
3. Regularized logistic regression with age splines and one-hot categoricals
|
||||
4. A boosted-tree model
|
||||
5. Inspection-only versus inspection-plus-DMV ablation
|
||||
|
||||
Keep validation and test sets at natural prevalence. If pass rows are sampled
|
||||
for training, preserve sampling probabilities and recalibrate on the untouched
|
||||
2024 partition.
|
||||
|
||||
## Metrics
|
||||
|
||||
Headline binary metrics:
|
||||
|
||||
- non-pass PR-AUC;
|
||||
- Brier score and log loss;
|
||||
- calibration intercept, slope, and reliability curve;
|
||||
- precision, recall, and lift at fixed review capacities; and
|
||||
- ROC-AUC as secondary context.
|
||||
|
||||
Use vehicle-clustered bootstrap confidence intervals. Report results by county,
|
||||
source era, vehicle-age band, prior outcome, fuel, history depth, and DMV
|
||||
match/staleness. Four-class analysis adds class-specific and macro PR-AUC,
|
||||
multiclass log loss/Brier score, calibration, and a confusion matrix.
|
||||
|
||||
## Drift contract
|
||||
|
||||
Monitor monthly/source-level volume, label recognition, outcome prevalence,
|
||||
blank rate, source/program mix, missingness, DMV staleness, unseen categories,
|
||||
prediction distribution, PR-AUC, Brier score, and calibration. Treat the
|
||||
`slc`→`slco` transition, new rich feeds, DMV 2021 gap, DMV 2024 endpoint, and
|
||||
partial 2026 period as explicit stress cases rather than ordinary random drift.
|
||||
122
docs/project_charter.md
Normal file
122
docs/project_charter.md
Normal file
@ -0,0 +1,122 @@
|
||||
# Utah Vehicle Health project charter
|
||||
|
||||
## Working title
|
||||
|
||||
**Utah Vehicle Health**
|
||||
|
||||
*What millions of emissions inspections reveal about the next test across the
|
||||
Utah county feeds available in this dataset.*
|
||||
|
||||
## Product promise
|
||||
|
||||
Explain how vehicle age, type, location, and prior inspection history relate to
|
||||
the chance of passing the first attempt of the next emissions-inspection
|
||||
episode.
|
||||
|
||||
The product measures **emissions-inspection outcomes**. It must not describe its
|
||||
score as a diagnosis of overall mechanical reliability, roadworthiness, safety,
|
||||
or legal compliance.
|
||||
|
||||
## Primary research question
|
||||
|
||||
> Using only information available before an inspection episode begins, how
|
||||
> accurately and reliably can we estimate whether its first attempt will pass?
|
||||
|
||||
An episode groups attempts for the same internal vehicle token when the gap
|
||||
between consecutive attempts is 30 days or less. The target is the first attempt
|
||||
of a new episode, not every rapid retest.
|
||||
|
||||
## Primary target
|
||||
|
||||
- `0 — pass`: normalized `PASS` or `P`
|
||||
- `1 — non-pass`: normalized `FAIL`, `F`, `REJECT`, or `ABORT`
|
||||
- Blank, null, and unrecognized results are unlabeled. They remain in chronology
|
||||
and coverage reporting but are excluded as supervised targets.
|
||||
|
||||
Because reject and abort can reflect process/readiness problems rather than
|
||||
mechanical failure, the project will also report:
|
||||
|
||||
- a four-class pass/fail/reject/abort analysis; and
|
||||
- a fail-versus-pass sensitivity analysis that excludes reject and abort.
|
||||
|
||||
## Scope
|
||||
|
||||
The returning-vehicle MVP uses universal inspection-history fields and prior
|
||||
DMV information where it is point-in-time valid. It covers participating
|
||||
inspection county/source feeds, not all 29 Utah counties.
|
||||
|
||||
The rich OBD/odometer fields are a later, source-specific extension for newer
|
||||
`slco`, `davis`, and `cache` records. They are not part of the statewide-style
|
||||
historical baseline.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. A reproducible, read-only extraction and private pseudonymization pipeline.
|
||||
2. A point-in-time episode/feature mart with an auditable exclusion report.
|
||||
3. Prevalence, previous-outcome, and logistic-regression baselines.
|
||||
4. One calibrated boosted-tree model and an inspection-only versus DMV-enhanced
|
||||
ablation.
|
||||
5. A locked temporal evaluation with subgroup and source-era diagnostics.
|
||||
6. Versioned, suppressed public aggregates for a four-page Bolt dashboard.
|
||||
7. A model card and data/methodology page documenting limitations.
|
||||
|
||||
## Headline success criteria
|
||||
|
||||
- Beat both the training-prevalence and previous-episode-outcome baselines on
|
||||
2025 PR-AUC and Brier score.
|
||||
- Produce calibrated probabilities, not just class labels.
|
||||
- Report performance by county/source era, vehicle-age band, history depth, and
|
||||
DMV match/staleness.
|
||||
- Reproduce all published charts from versioned sanitized outputs.
|
||||
- Export no VIN, plate, ZIP, station, raw JSON, operational record, or
|
||||
row-level prediction to Bolt.
|
||||
- Suppress public cells with fewer than 100 eligible inspections or fewer than
|
||||
10 observations in an outcome or its complement, with complementary
|
||||
suppression where totals could reveal a hidden cell.
|
||||
|
||||
## Initial feasibility result
|
||||
|
||||
Under label contract v3, the fixed-seed aggregate-only query in
|
||||
`sql/10_episode_cohort_feasibility.sql` produced 8,916 eligible
|
||||
returning-vehicle episodes from 1,730 of 2,000 sampled vehicle histories. The
|
||||
first-attempt non-pass rate was 11.80%, and the median gap from the prior
|
||||
episode was about 372 days. Of those targets, 1,732 use the documented Utah OBD
|
||||
binary proxy because that source leaves its overall-result field blank. This
|
||||
supports both the episode definition and a calibrated binary model. The sample
|
||||
is for pipeline feasibility, not a population estimate.
|
||||
|
||||
The 2025 sample showed a longer median gap and different outcome mix, reinforcing
|
||||
the need for source-era drift reporting and a locked chronological test.
|
||||
|
||||
## Development holdout status
|
||||
|
||||
During live verification, the explicit 2025 gate was opened once on the
|
||||
page-sampled development extract after both candidate specifications had been
|
||||
fixed from pre-2025 data. No test-informed model change was made. The frozen
|
||||
logistic model remains selected, and a later complete-data 2025 analysis will
|
||||
be treated as confirmatory rather than described as a pristine unseen test.
|
||||
The audit trail and results are recorded in
|
||||
[development_results.md](development_results.md).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Diagnosing an individual vehicle
|
||||
- Certifying that a vehicle will pass
|
||||
- Ranking or accusing inspection stations
|
||||
- Identifying owners or accepting VIN/plate input
|
||||
- Making causal claims about county programs
|
||||
- Treating missing outcomes as passes
|
||||
|
||||
## Staged build
|
||||
|
||||
1. **Foundation:** episode definition, label normalization, exclusions, secure
|
||||
extraction, and coverage checks.
|
||||
2. **Baseline:** inspection-history feature mart and transparent baselines.
|
||||
3. **Enrichment:** canonical vehicle dimension and point-in-time DMV features.
|
||||
4. **Modeling:** tree model, calibration, locked test, drift and subgroup audits.
|
||||
5. **Product:** sanitized aggregate export and Bolt MVP.
|
||||
6. **Stretch:** failure-to-pass journeys, multiclass probabilities, and a
|
||||
clearly scoped rich-OBD model.
|
||||
|
||||
See [modeling_protocol.md](modeling_protocol.md) and
|
||||
[dashboard_spec.md](dashboard_spec.md) for the detailed contracts.
|
||||
161
docs/project_options.md
Normal file
161
docs/project_options.md
Normal file
@ -0,0 +1,161 @@
|
||||
# Project options based on the countydata inventory
|
||||
|
||||
Inventory date: 2026-07-15
|
||||
|
||||
> **Selected:** Option 1, Utah Vehicle Health. The implementation contract is in
|
||||
> [project_charter.md](project_charter.md).
|
||||
|
||||
## What is actually available
|
||||
|
||||
The useful analytical source is the `countydata` PostgreSQL database, which is
|
||||
about 93.4 GB. Its normalized tables are projections of two large logical
|
||||
datasets, so their row counts should not be added together:
|
||||
|
||||
- 18,009,278 DMV registration/tax records with county, registration date,
|
||||
make, model, model year, fuel type, registration type/place, expiration and
|
||||
emissions dates, ZIP codes, and temporary-registration status.
|
||||
- 19,357,287 inspection records with county/source, timestamp, make, model,
|
||||
model year, station, result, OBD result, test/program type, and OBD summary.
|
||||
- Raw inspection JSON adds odometer, fuel, engine, transmission, vehicle type,
|
||||
readiness monitors, visual checks, DTCs, PIDs, and communication protocol for
|
||||
newer county/source feeds.
|
||||
|
||||
The vehicle histories link well without exposing identifiers: 88.2% of a
|
||||
10,000-VIN inspection sample had DMV history, and the sampled median was eight
|
||||
inspection visits per vehicle. In the other direction, 74.2% of sampled DMV
|
||||
vehicles had an inspection match.
|
||||
|
||||
Important limitations:
|
||||
|
||||
- DMV coverage is strong in 2016-2020 and 2022 through March 2024, but only 117
|
||||
rows are dated 2021. Treat 2024 as partial.
|
||||
- Inspection coverage is strong from 2010 through June 2026, but the four rows
|
||||
dated 1990 are clear date outliers.
|
||||
- Raw inspection `overall_result` labels are imbalanced: 71.06% pass, 3.61%
|
||||
fail, 3.56% reject, 2.31% abort, and 19.45% blank/null. Most missing values
|
||||
come from the older Utah County feed; its separately encoded OBD result can
|
||||
support a provenance-tagged binary target, but not the four-class analysis.
|
||||
- Rich OBD/odometer fields are source-dependent. They are essentially absent
|
||||
from the older `slc`, `utah`, and `weber` feeds and concentrated in `slco`,
|
||||
`davis`, and `cache` records from late 2024 onward.
|
||||
- Categories require cleaning: fuel values differ by case and punctuation, and
|
||||
inspection make values include aliases such as `TOYOTA`/`TOYOT` and
|
||||
`CHEVROLET`/`CHEVR`.
|
||||
|
||||
## Ranked ideas
|
||||
|
||||
### 1. Utah Vehicle Health and Reliability Observatory
|
||||
|
||||
Build a model that estimates the chance a vehicle will fail, reject, or abort
|
||||
its next emissions inspection using only information known before that test:
|
||||
vehicle age, canonical make/model, county, season, fuel, prior test outcomes,
|
||||
time since the previous test, and longitudinal DMV history.
|
||||
|
||||
The dashboard could include:
|
||||
|
||||
- make/model/year reliability scorecards with uncertainty intervals;
|
||||
- risk-versus-age curves and county comparisons;
|
||||
- prior-failure and repeat-test patterns;
|
||||
- calibrated individual what-if estimates without accepting or displaying a
|
||||
VIN; and
|
||||
- a methodology/data-quality page showing drift and missingness.
|
||||
|
||||
This is the strongest overall option because it combines SQL/data engineering,
|
||||
entity resolution, longitudinal feature engineering, classification or
|
||||
survival analysis, model explainability, and a compelling public dashboard.
|
||||
|
||||
Guardrail: do not use same-test OBD result, DTC count, or overall result as
|
||||
features when claiming to predict an outcome before the test. That would be
|
||||
target leakage. Use a time-based holdout and report PR-AUC, calibration, and
|
||||
Brier score rather than accuracy alone.
|
||||
|
||||
### 2. Failure-to-Pass Journey Analyzer
|
||||
|
||||
Follow vehicles forward after a fail or reject and model how many attempts and
|
||||
how much time it takes to achieve a pass. Compare journeys by vehicle age,
|
||||
canonical make/model, program, county/source, and failure history using
|
||||
time-to-event or competing-risk methods.
|
||||
|
||||
The dashboard could use journey funnels, transition diagrams, survival curves,
|
||||
and a cohort comparison tool. This is an unusually good fit for the repeated
|
||||
histories and can become a major page within option 1 instead of a separate app.
|
||||
|
||||
### 3. Utah EV and Hybrid Transition Atlas
|
||||
|
||||
Deduplicate repeated registrations into vehicle-by-period snapshots, estimate
|
||||
electric and hybrid share by county and ZIP, measure transitions in the vehicle
|
||||
fleet, and forecast adoption scenarios.
|
||||
|
||||
The dashboard could use a choropleth, adoption curves, county rankings, and a
|
||||
make/model explorer. This is visually strong and easy to explain, but the DMV
|
||||
2021 gap and March 2024 endpoint make honest uncertainty and partial-period
|
||||
handling essential.
|
||||
|
||||
### 4. OBD Early-Warning Lab
|
||||
|
||||
Scope the analysis to the richer `slco`, `davis`, and `cache` feeds. Use
|
||||
odometer, vehicle age, fuel, engine, readiness monitors, communication protocol,
|
||||
MIL state, and prior history to distinguish likely pass, fail, reject, and
|
||||
abort outcomes and discover common failure signatures.
|
||||
|
||||
The dashboard could show diagnostic pathways, readiness-monitor patterns, and
|
||||
failure signatures by vehicle cohort. It is technically novel, but it is not a
|
||||
statewide study because rich-feature coverage begins mainly in 2024-2025.
|
||||
|
||||
### 5. Risk-Adjusted Inspection Station Consistency
|
||||
|
||||
Fit a hierarchical model of station outcomes after controlling for vehicle age,
|
||||
make/model, program, county, and time. Use empirical-Bayes shrinkage or control
|
||||
charts to flag unusual reject, abort, or failure rates and detect process drift.
|
||||
|
||||
This is excellent applied statistics and anomaly detection. Public results
|
||||
should anonymize station identifiers, suppress small groups, show uncertainty,
|
||||
and describe anomalies as review signals rather than evidence of misconduct.
|
||||
|
||||
### 6. AutoClean: Government Vehicle Data Entity Resolution
|
||||
|
||||
Use the high VIN match rate as weak supervision to learn canonical make/model
|
||||
mappings between messy inspection strings and cleaner DMV values. Compare
|
||||
rules, fuzzy matching, and a supervised ranking model, then quantify how much
|
||||
normalization improves downstream analytics.
|
||||
|
||||
The dashboard could show before/after category fragmentation, match confidence,
|
||||
and correction examples using non-identifying values. This is a particularly
|
||||
strong data-engineering project and could also serve as the first pipeline stage
|
||||
for option 1.
|
||||
|
||||
### 7. Inspection Demand Forecast and Operations Dashboard
|
||||
|
||||
Forecast daily or weekly test volume by county and station using timestamps,
|
||||
seasonality, holidays, long-term trend, and recent history. Add change-point and
|
||||
volume-anomaly detection.
|
||||
|
||||
The dashboard could show workload forecasts, day/hour heatmaps, forecast
|
||||
intervals, and historical disruptions. It is feasible and useful, although it
|
||||
has less machine-learning depth than the reliability project unless forecasting
|
||||
and anomaly evaluation are developed carefully.
|
||||
|
||||
### 8. Vehicle Longevity and Survival Index
|
||||
|
||||
Treat the repeated DMV and inspection histories as censored longitudinal data.
|
||||
Estimate how long makes, models, fuel types, and model-year cohorts remain
|
||||
active, using Kaplan-Meier curves and a Cox or gradient-boosted survival model.
|
||||
|
||||
The dashboard could answer "Which vehicles stay on Utah roads the longest?"
|
||||
The main methodological challenge is that disappearance from the data can mean
|
||||
sale, relocation, incomplete coverage, or retirement, so the result must be
|
||||
described as observed-system retention rather than mechanical lifespan.
|
||||
|
||||
## Recommended project shape
|
||||
|
||||
Use option 1 as the main story and option 6 as its data-engineering foundation:
|
||||
|
||||
1. Build an incremental, read-only SQL extraction and canonical vehicle model.
|
||||
2. Create one row per eligible upcoming inspection using only prior information.
|
||||
3. Compare a transparent logistic baseline with a tree model.
|
||||
4. Validate on a later time period and separately by county/source.
|
||||
5. Export only aggregate scorecards, curves, and de-identified model outputs.
|
||||
6. Build the public Bolt dashboard over those safe outputs.
|
||||
|
||||
This gives the project a coherent end-to-end narrative across data engineering,
|
||||
analytics, machine learning, responsible validation, and product design.
|
||||
7
requirements.txt
Normal file
7
requirements.txt
Normal file
@ -0,0 +1,7 @@
|
||||
duckdb==1.4.5
|
||||
joblib==1.5.3
|
||||
numpy==2.0.2
|
||||
psycopg2-binary==2.9.12
|
||||
scipy==1.13.1
|
||||
scikit-learn==1.6.1
|
||||
threadpoolctl==3.6.0
|
||||
727
scripts/build_feature_mart.py
Normal file
727
scripts/build_feature_mart.py
Normal file
@ -0,0 +1,727 @@
|
||||
"""Build a private, point-in-time inspection feature mart with DuckDB.
|
||||
|
||||
Inputs are either a contiguous set of bounded monthly CSV.gz exports or the
|
||||
single complete-history development sample. Every input must have the manifest
|
||||
written by the corresponding secure exporter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
import duckdb
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PRIVATE_DATA_ROOT = (PROJECT_ROOT / "data/private").resolve()
|
||||
DEFAULT_INPUT = PRIVATE_DATA_ROOT / "inspection_batches"
|
||||
DEFAULT_DATABASE = PRIVATE_DATA_ROOT / "warehouse/vehicle_health.duckdb"
|
||||
DEFAULT_OUTPUT = PRIVATE_DATA_ROOT / "marts/inspection_feature_mart.parquet"
|
||||
SQL_ROOT = PROJECT_ROOT / "sql/local"
|
||||
|
||||
EXPECTED_COLUMNS = (
|
||||
"internal_event_id",
|
||||
"vehicle_token",
|
||||
"vehicle_bucket",
|
||||
"event_ts",
|
||||
"source_era",
|
||||
"public_county",
|
||||
"canonical_outcome",
|
||||
"outcome_label_source",
|
||||
"program_type",
|
||||
"test_type",
|
||||
"observed_make",
|
||||
"observed_model",
|
||||
"observed_model_year",
|
||||
)
|
||||
BOUNDED_KIND = "bounded_batch"
|
||||
DEVELOPMENT_KIND = "vehicle_history_development_sample"
|
||||
HEX_64 = re.compile(r"^[0-9a-f]{64}$")
|
||||
FINGERPRINT = re.compile(r"^[0-9a-f]{16,64}$")
|
||||
MEMORY_LIMIT = re.compile(r"^[1-9][0-9]*(?:MB|GB|TB)$", re.IGNORECASE)
|
||||
|
||||
SQL_FILES = (
|
||||
"20_stage_events.sql",
|
||||
"21_build_episodes.sql",
|
||||
"22_build_features.sql",
|
||||
"23_validate_mart.sql",
|
||||
)
|
||||
|
||||
CREATE_STAGING_SQL = """
|
||||
CREATE TABLE stg_events (
|
||||
batch_file VARCHAR NOT NULL,
|
||||
batch_kind VARCHAR NOT NULL,
|
||||
batch_start TIMESTAMP NOT NULL,
|
||||
batch_end TIMESTAMP NOT NULL,
|
||||
internal_event_id VARCHAR,
|
||||
vehicle_token VARCHAR,
|
||||
vehicle_bucket VARCHAR,
|
||||
event_ts VARCHAR,
|
||||
source_era VARCHAR,
|
||||
public_county VARCHAR,
|
||||
canonical_outcome VARCHAR,
|
||||
outcome_label_source VARCHAR,
|
||||
program_type VARCHAR,
|
||||
test_type VARCHAR,
|
||||
observed_make VARCHAR,
|
||||
observed_model VARCHAR,
|
||||
observed_model_year VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
# The dialect and every input column are explicit. In particular, do not let a
|
||||
# sample without quoted values convince DuckDB that quote handling is disabled.
|
||||
INGEST_SQL = """
|
||||
INSERT INTO stg_events
|
||||
SELECT
|
||||
? AS batch_file,
|
||||
? AS batch_kind,
|
||||
?::TIMESTAMP AS batch_start,
|
||||
?::TIMESTAMP AS batch_end,
|
||||
source.internal_event_id,
|
||||
source.vehicle_token,
|
||||
source.vehicle_bucket,
|
||||
source.event_ts,
|
||||
source.source_era,
|
||||
source.public_county,
|
||||
source.canonical_outcome,
|
||||
source.outcome_label_source,
|
||||
source.program_type,
|
||||
source.test_type,
|
||||
source.observed_make,
|
||||
source.observed_model,
|
||||
source.observed_model_year
|
||||
FROM read_csv(
|
||||
?,
|
||||
header = true,
|
||||
auto_detect = false,
|
||||
delim = ',',
|
||||
quote = '"',
|
||||
escape = '"',
|
||||
nullstr = '',
|
||||
strict_mode = true,
|
||||
columns = {
|
||||
'internal_event_id': 'VARCHAR',
|
||||
'vehicle_token': 'VARCHAR',
|
||||
'vehicle_bucket': 'VARCHAR',
|
||||
'event_ts': 'VARCHAR',
|
||||
'source_era': 'VARCHAR',
|
||||
'public_county': 'VARCHAR',
|
||||
'canonical_outcome': 'VARCHAR',
|
||||
'outcome_label_source': 'VARCHAR',
|
||||
'program_type': 'VARCHAR',
|
||||
'test_type': 'VARCHAR',
|
||||
'observed_make': 'VARCHAR',
|
||||
'observed_model': 'VARCHAR',
|
||||
'observed_model_year': 'VARCHAR'
|
||||
}
|
||||
) AS source
|
||||
"""
|
||||
|
||||
|
||||
class MartBuildError(RuntimeError):
|
||||
"""Raised when private input or a mart invariant fails closed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InputBatch:
|
||||
path: Path
|
||||
manifest_path: Path
|
||||
export_kind: str
|
||||
start: datetime
|
||||
end: datetime
|
||||
rows: int
|
||||
query_version: str
|
||||
query_sha256: str
|
||||
key_version: str
|
||||
key_fingerprint: str
|
||||
compressed_sha256: str
|
||||
compressed_bytes: int
|
||||
manifest: Mapping[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuildConfig:
|
||||
input_path: Path = DEFAULT_INPUT
|
||||
database_path: Path = DEFAULT_DATABASE
|
||||
output_path: Path = DEFAULT_OUTPUT
|
||||
episode_gap_days: int = 30
|
||||
memory_limit: str = "4GB"
|
||||
threads: int = max(1, min(4, os.cpu_count() or 1))
|
||||
allow_gaps: bool = False
|
||||
allow_external_output: bool = False
|
||||
overwrite: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuildSummary:
|
||||
database_path: Path
|
||||
output_path: Path
|
||||
manifest_path: Path
|
||||
input_rows: int
|
||||
clean_events: int
|
||||
episodes: int
|
||||
mart_rows: int
|
||||
eligible_rows: int
|
||||
source_data_kind: str
|
||||
population_estimate_allowed: bool
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def parse_manifest_timestamp(value: object, field: str) -> datetime:
|
||||
if not isinstance(value, str):
|
||||
raise MartBuildError(f"Manifest field {field!r} must be an ISO timestamp")
|
||||
encoded = value[:-1] + "+00:00" if value.endswith("Z") else value
|
||||
try:
|
||||
parsed = datetime.fromisoformat(encoded)
|
||||
except ValueError as exc:
|
||||
raise MartBuildError(f"Manifest field {field!r} is not ISO-8601") from exc
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
return parsed
|
||||
|
||||
|
||||
def _required_text(manifest: Mapping[str, Any], field: str) -> str:
|
||||
value = manifest.get(field)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise MartBuildError(f"Manifest field {field!r} must be non-empty text")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _required_nonnegative_int(manifest: Mapping[str, Any], field: str) -> int:
|
||||
value = manifest.get(field)
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise MartBuildError(f"Manifest field {field!r} must be a nonnegative integer")
|
||||
return value
|
||||
|
||||
|
||||
def _read_header(path: Path) -> Tuple[str, ...]:
|
||||
try:
|
||||
with gzip.open(path, "rt", encoding="utf-8", newline="") as handle:
|
||||
row = next(csv.reader(handle), None)
|
||||
except (OSError, UnicodeError, csv.Error) as exc:
|
||||
raise MartBuildError(f"Cannot read gzip CSV header: {path}") from exc
|
||||
if row is None:
|
||||
raise MartBuildError(f"Input gzip CSV is empty: {path}")
|
||||
return tuple(row)
|
||||
|
||||
|
||||
def _load_batch(path: Path) -> InputBatch:
|
||||
manifest_path = path.with_name(path.name + ".manifest.json")
|
||||
if not manifest_path.is_file():
|
||||
raise MartBuildError(f"Missing paired manifest: {manifest_path}")
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise MartBuildError(f"Cannot parse manifest: {manifest_path}") from exc
|
||||
if not isinstance(manifest, dict):
|
||||
raise MartBuildError(f"Manifest must contain a JSON object: {manifest_path}")
|
||||
|
||||
columns = manifest.get("columns")
|
||||
if not isinstance(columns, (list, tuple)) or tuple(columns) != EXPECTED_COLUMNS:
|
||||
raise MartBuildError(f"Unexpected manifest columns: {manifest_path}")
|
||||
if _read_header(path) != EXPECTED_COLUMNS:
|
||||
raise MartBuildError(f"CSV header does not match the private schema: {path}")
|
||||
|
||||
export_kind = _required_text(manifest, "export_kind")
|
||||
if export_kind not in (BOUNDED_KIND, DEVELOPMENT_KIND):
|
||||
raise MartBuildError(f"Unsupported export_kind {export_kind!r}: {manifest_path}")
|
||||
start = parse_manifest_timestamp(
|
||||
manifest.get("source_start_inclusive"), "source_start_inclusive"
|
||||
)
|
||||
end = parse_manifest_timestamp(
|
||||
manifest.get("source_end_exclusive"), "source_end_exclusive"
|
||||
)
|
||||
if end <= start:
|
||||
raise MartBuildError(f"Manifest source interval is empty or reversed: {path}")
|
||||
|
||||
rows = _required_nonnegative_int(manifest, "rows")
|
||||
compressed_bytes = _required_nonnegative_int(manifest, "compressed_file_bytes")
|
||||
actual_bytes = path.stat().st_size
|
||||
if compressed_bytes != actual_bytes:
|
||||
raise MartBuildError(
|
||||
f"Compressed byte count mismatch for {path}: "
|
||||
f"manifest={compressed_bytes}, actual={actual_bytes}"
|
||||
)
|
||||
compressed_sha256 = _required_text(manifest, "compressed_file_sha256").lower()
|
||||
if not HEX_64.fullmatch(compressed_sha256):
|
||||
raise MartBuildError(f"Invalid compressed SHA-256 in {manifest_path}")
|
||||
actual_sha256 = file_sha256(path)
|
||||
if compressed_sha256 != actual_sha256:
|
||||
raise MartBuildError(f"Compressed SHA-256 mismatch for {path}")
|
||||
|
||||
query_sha256 = _required_text(manifest, "query_sha256").lower()
|
||||
if not HEX_64.fullmatch(query_sha256):
|
||||
raise MartBuildError(f"Invalid query SHA-256 in {manifest_path}")
|
||||
key_fingerprint = _required_text(
|
||||
manifest, "vehicle_token_key_fingerprint"
|
||||
).lower()
|
||||
if not FINGERPRINT.fullmatch(key_fingerprint):
|
||||
raise MartBuildError(f"Invalid key fingerprint in {manifest_path}")
|
||||
|
||||
if export_kind == BOUNDED_KIND:
|
||||
if manifest.get("classification") != (
|
||||
"private_pseudonymized_analytical_staging"
|
||||
):
|
||||
raise MartBuildError(f"Unexpected bounded-batch classification: {path}")
|
||||
else:
|
||||
if manifest.get("classification") != (
|
||||
"private_pseudonymized_development_only"
|
||||
):
|
||||
raise MartBuildError(f"Unexpected development classification: {path}")
|
||||
if manifest.get("population_estimate_allowed") is not False:
|
||||
raise MartBuildError(
|
||||
"Development history manifest must prohibit population estimates"
|
||||
)
|
||||
|
||||
return InputBatch(
|
||||
path=path,
|
||||
manifest_path=manifest_path,
|
||||
export_kind=export_kind,
|
||||
start=start,
|
||||
end=end,
|
||||
rows=rows,
|
||||
query_version=_required_text(manifest, "query_version"),
|
||||
query_sha256=query_sha256,
|
||||
key_version=_required_text(manifest, "vehicle_token_key_version"),
|
||||
key_fingerprint=key_fingerprint,
|
||||
compressed_sha256=compressed_sha256,
|
||||
compressed_bytes=compressed_bytes,
|
||||
manifest=manifest,
|
||||
)
|
||||
|
||||
|
||||
def _discover_files(input_path: Path) -> List[Path]:
|
||||
if input_path.is_file():
|
||||
if not input_path.name.endswith(".csv.gz"):
|
||||
raise MartBuildError("A feature-mart input file must end in .csv.gz")
|
||||
return [input_path]
|
||||
if not input_path.is_dir():
|
||||
raise MartBuildError(f"Input path does not exist: {input_path}")
|
||||
paths = sorted(path for path in input_path.glob("*.csv.gz") if path.is_file())
|
||||
if not paths:
|
||||
raise MartBuildError(f"No .csv.gz inputs found in {input_path}")
|
||||
return paths
|
||||
|
||||
|
||||
def discover_and_validate_batches(config: BuildConfig) -> List[InputBatch]:
|
||||
input_path = config.input_path.resolve()
|
||||
if not config.allow_external_output and not input_path.is_relative_to(
|
||||
PRIVATE_DATA_ROOT
|
||||
):
|
||||
raise MartBuildError(f"Private inputs must stay under {PRIVATE_DATA_ROOT}")
|
||||
|
||||
batches = [_load_batch(path.resolve()) for path in _discover_files(input_path)]
|
||||
kinds = {batch.export_kind for batch in batches}
|
||||
if len(kinds) != 1:
|
||||
raise MartBuildError("Do not mix bounded batches and development samples")
|
||||
kind = next(iter(kinds))
|
||||
|
||||
key_versions = {batch.key_version for batch in batches}
|
||||
fingerprints = {batch.key_fingerprint for batch in batches}
|
||||
if len(key_versions) != 1 or len(fingerprints) != 1:
|
||||
raise MartBuildError("All inputs must use one VIN-token key and version")
|
||||
|
||||
if kind == DEVELOPMENT_KIND:
|
||||
if len(batches) != 1:
|
||||
raise MartBuildError("A development mart accepts exactly one history sample")
|
||||
return batches
|
||||
|
||||
query_versions = {batch.query_version for batch in batches}
|
||||
query_hashes = {batch.query_sha256 for batch in batches}
|
||||
if len(query_versions) != 1 or len(query_hashes) != 1:
|
||||
raise MartBuildError("All bounded batches must use one extraction query version")
|
||||
|
||||
batches.sort(key=lambda batch: (batch.start, batch.end, str(batch.path)))
|
||||
for previous, current in zip(batches, batches[1:]):
|
||||
if current.start < previous.end:
|
||||
raise MartBuildError(
|
||||
f"Source intervals overlap: {previous.path.name} and "
|
||||
f"{current.path.name}"
|
||||
)
|
||||
if current.start > previous.end and not config.allow_gaps:
|
||||
raise MartBuildError(
|
||||
f"Source interval gap between {previous.end.isoformat()} and "
|
||||
f"{current.start.isoformat()}; use --allow-gaps only for an "
|
||||
"explicitly incomplete analytical build"
|
||||
)
|
||||
return batches
|
||||
|
||||
|
||||
def _validate_config(config: BuildConfig) -> BuildConfig:
|
||||
if not 1 <= config.episode_gap_days <= 365:
|
||||
raise MartBuildError("episode_gap_days must be between 1 and 365")
|
||||
if not 1 <= config.threads <= 64:
|
||||
raise MartBuildError("threads must be between 1 and 64")
|
||||
if not MEMORY_LIMIT.fullmatch(config.memory_limit):
|
||||
raise MartBuildError("memory_limit must look like 4096MB or 4GB")
|
||||
|
||||
database_path = config.database_path.resolve()
|
||||
output_path = config.output_path.resolve()
|
||||
if output_path.suffix.lower() != ".parquet":
|
||||
raise MartBuildError("Feature-mart output must use a .parquet suffix")
|
||||
if not config.allow_external_output:
|
||||
for path in (database_path, output_path):
|
||||
if not path.is_relative_to(PRIVATE_DATA_ROOT):
|
||||
raise MartBuildError(f"Private outputs must stay under {PRIVATE_DATA_ROOT}")
|
||||
if database_path == output_path:
|
||||
raise MartBuildError("Database and Parquet output paths must differ")
|
||||
|
||||
return BuildConfig(
|
||||
input_path=config.input_path.resolve(),
|
||||
database_path=database_path,
|
||||
output_path=output_path,
|
||||
episode_gap_days=config.episode_gap_days,
|
||||
memory_limit=config.memory_limit.upper(),
|
||||
threads=config.threads,
|
||||
allow_gaps=config.allow_gaps,
|
||||
allow_external_output=config.allow_external_output,
|
||||
overwrite=config.overwrite,
|
||||
)
|
||||
|
||||
|
||||
def _load_sql(name: str) -> str:
|
||||
path = SQL_ROOT / name
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise MartBuildError(f"Cannot read local transform SQL: {path}") from exc
|
||||
|
||||
|
||||
def _remove_partial(paths: Iterable[Path]) -> None:
|
||||
for path in paths:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _output_columns(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, str]]:
|
||||
rows = connection.execute("DESCRIBE SELECT * FROM feature_mart").fetchall()
|
||||
return [
|
||||
{"name": str(row[0]), "duckdb_type": str(row[1])}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def build_feature_mart(config: BuildConfig) -> BuildSummary:
|
||||
config = _validate_config(config)
|
||||
batches = discover_and_validate_batches(config)
|
||||
source_data_kind = batches[0].export_kind
|
||||
population_estimate_allowed = (
|
||||
source_data_kind == BOUNDED_KIND and not config.allow_gaps
|
||||
)
|
||||
|
||||
database_path = config.database_path
|
||||
output_path = config.output_path
|
||||
manifest_path = output_path.with_name(output_path.name + ".manifest.json")
|
||||
partial_database = database_path.with_name(database_path.name + ".partial")
|
||||
partial_output = output_path.with_name(output_path.name + ".partial")
|
||||
partial_manifest = manifest_path.with_name(manifest_path.name + ".partial")
|
||||
partial_wal = partial_database.with_name(partial_database.name + ".wal")
|
||||
partial_paths = (partial_database, partial_output, partial_manifest, partial_wal)
|
||||
|
||||
finals = (database_path, output_path, manifest_path)
|
||||
existing = [path for path in finals if path.exists()]
|
||||
if existing and not config.overwrite:
|
||||
raise MartBuildError(
|
||||
"Refusing to overwrite existing outputs: "
|
||||
+ ", ".join(str(path) for path in existing)
|
||||
)
|
||||
|
||||
os.umask(0o077)
|
||||
database_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temp_directory = database_path.parent / ".duckdb_tmp"
|
||||
temp_directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
_remove_partial(partial_paths)
|
||||
|
||||
connection: Optional[duckdb.DuckDBPyConnection] = None
|
||||
try:
|
||||
connection = duckdb.connect(str(partial_database))
|
||||
connection.execute("SET threads = ?", [config.threads])
|
||||
connection.execute("SET memory_limit = ?", [config.memory_limit])
|
||||
connection.execute("SET temp_directory = ?", [str(temp_directory)])
|
||||
connection.execute("SET preserve_insertion_order = false")
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE build_config AS
|
||||
SELECT
|
||||
?::INTEGER AS episode_gap_days,
|
||||
?::VARCHAR AS source_data_kind,
|
||||
?::BOOLEAN AS population_estimate_allowed
|
||||
""",
|
||||
[
|
||||
config.episode_gap_days,
|
||||
source_data_kind,
|
||||
population_estimate_allowed,
|
||||
],
|
||||
)
|
||||
connection.execute(CREATE_STAGING_SQL)
|
||||
|
||||
for batch in batches:
|
||||
connection.execute(
|
||||
INGEST_SQL,
|
||||
[
|
||||
str(batch.path),
|
||||
batch.export_kind,
|
||||
batch.start.isoformat(sep=" "),
|
||||
batch.end.isoformat(sep=" "),
|
||||
str(batch.path),
|
||||
],
|
||||
)
|
||||
|
||||
actual_by_file = dict(
|
||||
connection.execute(
|
||||
"SELECT batch_file, count(*) FROM stg_events GROUP BY batch_file"
|
||||
).fetchall()
|
||||
)
|
||||
for batch in batches:
|
||||
actual = int(actual_by_file.get(str(batch.path), 0))
|
||||
if actual != batch.rows:
|
||||
raise MartBuildError(
|
||||
f"CSV row count mismatch for {batch.path}: "
|
||||
f"manifest={batch.rows}, actual={actual}"
|
||||
)
|
||||
|
||||
duplicate_ids = connection.execute(
|
||||
"""
|
||||
SELECT count(*)
|
||||
FROM (
|
||||
SELECT internal_event_id
|
||||
FROM stg_events
|
||||
GROUP BY internal_event_id
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
"""
|
||||
).fetchone()[0]
|
||||
if duplicate_ids:
|
||||
raise MartBuildError(
|
||||
f"Input contains {duplicate_ids} duplicated internal event IDs"
|
||||
)
|
||||
|
||||
inconsistent_buckets = connection.execute(
|
||||
"""
|
||||
SELECT count(*)
|
||||
FROM (
|
||||
SELECT lower(vehicle_token) AS vehicle_token
|
||||
FROM stg_events
|
||||
WHERE regexp_full_match(lower(vehicle_token), '^[0-9a-f]{64}$')
|
||||
AND try_cast(vehicle_bucket AS INTEGER) BETWEEN 0 AND 99
|
||||
GROUP BY lower(vehicle_token)
|
||||
HAVING count(DISTINCT try_cast(vehicle_bucket AS INTEGER)) > 1
|
||||
)
|
||||
"""
|
||||
).fetchone()[0]
|
||||
if inconsistent_buckets:
|
||||
raise MartBuildError(
|
||||
f"Input contains {inconsistent_buckets} vehicle tokens with "
|
||||
"inconsistent audit buckets"
|
||||
)
|
||||
|
||||
sql_hashes: Dict[str, str] = {}
|
||||
for name in SQL_FILES:
|
||||
sql = _load_sql(name)
|
||||
sql_hashes[name] = hashlib.sha256(sql.encode("utf-8")).hexdigest()
|
||||
connection.execute(sql)
|
||||
|
||||
violations = connection.execute(
|
||||
"""
|
||||
SELECT check_name, violation_count
|
||||
FROM mart_validation
|
||||
WHERE violation_count <> 0
|
||||
ORDER BY check_name
|
||||
"""
|
||||
).fetchall()
|
||||
if violations:
|
||||
details = ", ".join(f"{name}={count}" for name, count in violations)
|
||||
raise MartBuildError(f"Mart invariant failure: {details}")
|
||||
|
||||
counts = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
(SELECT count(*) FROM stg_events),
|
||||
(SELECT count(*) FROM clean_events),
|
||||
(SELECT count(*) FROM inspection_episodes),
|
||||
(SELECT count(*) FROM feature_mart),
|
||||
(SELECT count(*) FROM feature_mart
|
||||
WHERE eligible_returning_target)
|
||||
"""
|
||||
).fetchone()
|
||||
input_rows, clean_events, episodes, mart_rows, eligible_rows = map(
|
||||
int, counts
|
||||
)
|
||||
|
||||
columns = _output_columns(connection)
|
||||
connection.execute("ANALYZE")
|
||||
connection.execute(
|
||||
"""
|
||||
COPY (
|
||||
SELECT *
|
||||
FROM feature_mart
|
||||
ORDER BY vehicle_token, episode_number
|
||||
) TO ? (FORMAT PARQUET, COMPRESSION ZSTD)
|
||||
""",
|
||||
[str(partial_output)],
|
||||
)
|
||||
|
||||
build_id = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"inputs": [batch.compressed_sha256 for batch in batches],
|
||||
"sql": sql_hashes,
|
||||
"gap": config.episode_gap_days,
|
||||
},
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
output_sha256 = file_sha256(partial_output)
|
||||
build_manifest: Dict[str, Any] = {
|
||||
"build_kind": "private_leakage_safe_inspection_feature_mart",
|
||||
"build_id": build_id,
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"classification": "private_pseudonymized_analytical_mart",
|
||||
"source_data_kind": source_data_kind,
|
||||
"population_estimate_allowed": population_estimate_allowed,
|
||||
"episode_gap_days": config.episode_gap_days,
|
||||
"vehicle_quality_contract": (
|
||||
"eligibility uses only prior events: at most 50 prior events "
|
||||
"and at most 4 prior events on any day"
|
||||
),
|
||||
"outcome_label_source_contract": {
|
||||
"allowed_values": ["overall_result", "utah_obd_proxy"],
|
||||
"null_allowed_only_for_unlabeled_outcomes": True,
|
||||
"utah_obd_proxy_source_era": "utah",
|
||||
"feature_role": "audit_only_not_a_predictor",
|
||||
},
|
||||
"vin_audit_contract": "vehicle_bucket < 10",
|
||||
"inputs": [
|
||||
{
|
||||
"file": str(batch.path),
|
||||
"manifest": str(batch.manifest_path),
|
||||
"export_kind": batch.export_kind,
|
||||
"query_version": batch.query_version,
|
||||
"query_sha256": batch.query_sha256,
|
||||
"source_start_inclusive": batch.start.isoformat(),
|
||||
"source_end_exclusive": batch.end.isoformat(),
|
||||
"rows": batch.rows,
|
||||
"compressed_file_sha256": batch.compressed_sha256,
|
||||
}
|
||||
for batch in batches
|
||||
],
|
||||
"vehicle_token_key_version": batches[0].key_version,
|
||||
"vehicle_token_key_fingerprint": batches[0].key_fingerprint,
|
||||
"transform_sql_sha256": sql_hashes,
|
||||
"row_counts": {
|
||||
"input": input_rows,
|
||||
"clean_events": clean_events,
|
||||
"episodes": episodes,
|
||||
"feature_mart": mart_rows,
|
||||
"eligible_returning_targets": eligible_rows,
|
||||
},
|
||||
"columns": columns,
|
||||
"parquet_sha256": output_sha256,
|
||||
"parquet_bytes": partial_output.stat().st_size,
|
||||
}
|
||||
partial_manifest.write_text(
|
||||
json.dumps(build_manifest, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
connection.execute("CHECKPOINT")
|
||||
connection.close()
|
||||
connection = None
|
||||
|
||||
os.replace(partial_database, database_path)
|
||||
os.replace(partial_output, output_path)
|
||||
os.replace(partial_manifest, manifest_path)
|
||||
|
||||
return BuildSummary(
|
||||
database_path=database_path,
|
||||
output_path=output_path,
|
||||
manifest_path=manifest_path,
|
||||
input_rows=input_rows,
|
||||
clean_events=clean_events,
|
||||
episodes=episodes,
|
||||
mart_rows=mart_rows,
|
||||
eligible_rows=eligible_rows,
|
||||
source_data_kind=source_data_kind,
|
||||
population_estimate_allowed=population_estimate_allowed,
|
||||
)
|
||||
except Exception:
|
||||
if connection is not None:
|
||||
connection.close()
|
||||
_remove_partial(partial_paths)
|
||||
raise
|
||||
|
||||
|
||||
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
"--batch-dir",
|
||||
dest="input_path",
|
||||
type=Path,
|
||||
default=DEFAULT_INPUT,
|
||||
help="A bounded-batch directory or one development history CSV.gz.",
|
||||
)
|
||||
parser.add_argument("--database", type=Path, default=DEFAULT_DATABASE)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--episode-gap-days", type=int, default=30)
|
||||
parser.add_argument("--memory-limit", default="4GB")
|
||||
parser.add_argument(
|
||||
"--threads", type=int, default=max(1, min(4, os.cpu_count() or 1))
|
||||
)
|
||||
parser.add_argument("--allow-gaps", action="store_true")
|
||||
parser.add_argument("--allow-external-output", action="store_true")
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
config = BuildConfig(
|
||||
input_path=args.input_path,
|
||||
database_path=args.database,
|
||||
output_path=args.output,
|
||||
episode_gap_days=args.episode_gap_days,
|
||||
memory_limit=args.memory_limit,
|
||||
threads=args.threads,
|
||||
allow_gaps=args.allow_gaps,
|
||||
allow_external_output=args.allow_external_output,
|
||||
overwrite=args.overwrite,
|
||||
)
|
||||
try:
|
||||
summary = build_feature_mart(config)
|
||||
except (MartBuildError, duckdb.Error, OSError) as exc:
|
||||
print(f"Feature-mart build failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"Built {summary.mart_rows:,} episode rows "
|
||||
f"({summary.eligible_rows:,} eligible returning targets)"
|
||||
)
|
||||
print(f"Private DuckDB warehouse: {summary.database_path}")
|
||||
print(f"Private Parquet mart: {summary.output_path}")
|
||||
print(f"Build manifest: {summary.manifest_path}")
|
||||
if not summary.population_estimate_allowed:
|
||||
print("Development sample: population estimates are prohibited.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
96
scripts/check_connection.py
Normal file
96
scripts/check_connection.py
Normal file
@ -0,0 +1,96 @@
|
||||
"""Verify the countydata connection without exposing credentials or row data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
|
||||
REQUIRED_ENV_VARS = (
|
||||
"PGHOST",
|
||||
"PGPORT",
|
||||
"PGDATABASE",
|
||||
"PGUSER",
|
||||
"PGPASSWORD",
|
||||
"PGSSLMODE",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name)]
|
||||
if missing:
|
||||
print(
|
||||
"Missing database environment variables: " + ", ".join(missing),
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("Run this through the VS Code task so .env is loaded safely.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if os.environ["PGSSLMODE"].lower() not in {"require", "verify-ca", "verify-full"}:
|
||||
print("Refusing to connect because PGSSLMODE does not require TLS.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
connection = psycopg2.connect(
|
||||
application_name="summer_project_vscode_check",
|
||||
connect_timeout=10,
|
||||
options=(
|
||||
"-c default_transaction_read_only=on "
|
||||
"-c statement_timeout=30000"
|
||||
),
|
||||
)
|
||||
connection.set_session(readonly=True)
|
||||
except psycopg2.Error as exc:
|
||||
print(f"Connection failed: {exc.diag.message_primary or type(exc).__name__}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
with connection, connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
current_database(),
|
||||
current_user,
|
||||
current_setting('transaction_read_only'),
|
||||
version()
|
||||
"""
|
||||
)
|
||||
database, user, read_only, version = cursor.fetchone()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT ssl, version, cipher, bits
|
||||
FROM pg_stat_ssl
|
||||
WHERE pid = pg_backend_pid()
|
||||
"""
|
||||
)
|
||||
ssl, tls_version, cipher, bits = cursor.fetchone()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT count(*)
|
||||
FROM pg_class AS c
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_toast%'
|
||||
"""
|
||||
)
|
||||
relation_count = cursor.fetchone()[0]
|
||||
|
||||
print("Countydata connection succeeded")
|
||||
print(f" database: {database}")
|
||||
print(f" user: {user}")
|
||||
print(f" transaction read-only: {read_only}")
|
||||
print(f" TLS: {ssl} ({tls_version}, {cipher}, {bits}-bit)")
|
||||
print(f" visible non-system relations: {relation_count}")
|
||||
print(f" server: {version}")
|
||||
return 0
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
56
scripts/create_vin_hash_key.py
Normal file
56
scripts/create_vin_hash_key.py
Normal file
@ -0,0 +1,56 @@
|
||||
"""Create the local project-specific VIN HMAC key without printing it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import secrets
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_PATH = PROJECT_ROOT / ".secrets/vin_hmac.key"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--path", type=Path, default=DEFAULT_PATH)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def key_fingerprint(key: bytes) -> str:
|
||||
return hashlib.sha256(
|
||||
b"utah-vehicle-health/key-fingerprint/v1\0" + key
|
||||
).hexdigest()[:16]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = parse_args().path.resolve()
|
||||
os.umask(0o077)
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
if path.exists():
|
||||
mode = stat.S_IMODE(path.stat().st_mode)
|
||||
if mode & 0o077:
|
||||
print(f"Existing key has unsafe permissions: {path}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"Key already exists; leaving it unchanged: {path}")
|
||||
return 0
|
||||
|
||||
key = secrets.token_bytes(32)
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="ascii") as handle:
|
||||
handle.write(key.hex() + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
print(f"Created private VIN HMAC key: {path}")
|
||||
print(f"Non-secret key fingerprint: {key_fingerprint(key)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1205
scripts/export_dashboard_data.py
Normal file
1205
scripts/export_dashboard_data.py
Normal file
File diff suppressed because it is too large
Load Diff
234
scripts/export_history_sample.py
Normal file
234
scripts/export_history_sample.py
Normal file
@ -0,0 +1,234 @@
|
||||
"""Export complete histories for a private development-only vehicle sample.
|
||||
|
||||
The source sample is page-based and therefore not population-representative.
|
||||
Use it to develop and test the episode/model pipeline, never for final rates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import psycopg2
|
||||
|
||||
try:
|
||||
from . import export_inspection_batch as secure_export
|
||||
except ImportError: # Direct execution: python scripts/export_history_sample.py
|
||||
import export_inspection_batch as secure_export
|
||||
|
||||
|
||||
QUERY_VERSION = "inspection_history_development_sample_v3"
|
||||
|
||||
|
||||
def parse_timestamp(value: str) -> datetime:
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"Invalid ISO timestamp: {value}") from exc
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--vehicles", type=int, default=10_000)
|
||||
parser.add_argument("--sample-percent", type=float, default=0.5)
|
||||
parser.add_argument("--seed", type=int, default=20260715)
|
||||
parser.add_argument("--start", type=parse_timestamp, default=datetime(2010, 1, 1))
|
||||
parser.add_argument("--end", type=parse_timestamp, default=datetime(2026, 7, 1))
|
||||
parser.add_argument("--fetch-size", type=int, default=5_000)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_args(args: argparse.Namespace) -> Path:
|
||||
if not 100 <= args.vehicles <= 100_000:
|
||||
raise ValueError("--vehicles must be between 100 and 100000")
|
||||
if not 0 < args.sample_percent <= 5:
|
||||
raise ValueError("--sample-percent must be greater than 0 and at most 5")
|
||||
if not 100 <= args.fetch_size <= 50_000:
|
||||
raise ValueError("--fetch-size must be between 100 and 50000")
|
||||
if args.end <= args.start:
|
||||
raise ValueError("--end must be later than --start")
|
||||
|
||||
output = args.output
|
||||
if output is None:
|
||||
output = (
|
||||
secure_export.PRIVATE_DATA_ROOT
|
||||
/ "development"
|
||||
/ f"history_sample_{args.vehicles}.csv.gz"
|
||||
)
|
||||
output = output.resolve()
|
||||
if not output.is_relative_to(secure_export.PRIVATE_DATA_ROOT):
|
||||
raise ValueError(
|
||||
f"Development histories must stay under {secure_export.PRIVATE_DATA_ROOT}"
|
||||
)
|
||||
manifest = output.with_name(output.name + ".manifest.json")
|
||||
if (output.exists() or manifest.exists()) and not args.overwrite:
|
||||
raise FileExistsError(
|
||||
f"Refusing to overwrite {output} or its manifest; use --overwrite"
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def make_sql(sample_percent: float, seed: int) -> str:
|
||||
safe_percent = f"{sample_percent:.6f}"
|
||||
safe_seed = int(seed)
|
||||
return f"""
|
||||
WITH candidate_vins AS MATERIALIZED (
|
||||
SELECT DISTINCT vin
|
||||
FROM data.inspection_search
|
||||
TABLESAMPLE SYSTEM ({safe_percent}) REPEATABLE ({safe_seed})
|
||||
WHERE vin IS NOT NULL
|
||||
AND length(btrim(vin)) = 17
|
||||
),
|
||||
sampled_vins AS MATERIALIZED (
|
||||
SELECT vin
|
||||
FROM candidate_vins
|
||||
ORDER BY md5(vin)
|
||||
LIMIT %(vehicle_limit)s
|
||||
)
|
||||
SELECT
|
||||
s.id AS internal_event_id,
|
||||
s.vin,
|
||||
s.test_start AS event_ts,
|
||||
lower(btrim(s.county)) AS source_era,
|
||||
CASE
|
||||
WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake'
|
||||
ELSE lower(btrim(s.county))
|
||||
END AS public_county,
|
||||
{secure_export.OUTCOME_SELECT_SQL},
|
||||
lower(nullif(btrim(s.program_type), '')) AS program_type,
|
||||
upper(nullif(btrim(s.test_type), '')) AS test_type,
|
||||
upper(nullif(btrim(v.make), '')) AS observed_make,
|
||||
upper(nullif(btrim(v.model), '')) AS observed_model,
|
||||
v.year AS observed_model_year
|
||||
FROM data.inspection_search AS s
|
||||
JOIN data.inspection_vehicle AS v USING (id)
|
||||
JOIN sampled_vins ON sampled_vins.vin = s.vin
|
||||
WHERE s.test_start >= %(start_ts)s
|
||||
AND s.test_start < %(end_ts)s
|
||||
ORDER BY s.test_start, s.id
|
||||
"""
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
os.umask(0o077)
|
||||
try:
|
||||
output = validate_args(args)
|
||||
key, key_version, key_fingerprint = secure_export.get_hmac_configuration()
|
||||
except (FileExistsError, ValueError) as exc:
|
||||
print(f"Configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest = output.with_name(output.name + ".manifest.json")
|
||||
partial = output.with_name(output.name + ".partial")
|
||||
partial_manifest = manifest.with_name(manifest.name + ".partial")
|
||||
sql = make_sql(args.sample_percent, args.seed)
|
||||
|
||||
try:
|
||||
connection = secure_export.connect_read_only()
|
||||
except (ValueError, RuntimeError, psycopg2.Error) as exc:
|
||||
print(f"Secure read-only connection failed: {type(exc).__name__}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
source_rows_read = 0
|
||||
rows_written = 0
|
||||
invalid_vins_skipped = 0
|
||||
last_order_key: Optional[tuple[object, object]] = None
|
||||
try:
|
||||
with gzip.open(partial, "wt", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.writer(handle)
|
||||
writer.writerow(secure_export.OUTPUT_COLUMNS)
|
||||
with connection.cursor(name="uvh_history_sample") as cursor:
|
||||
cursor.itersize = args.fetch_size
|
||||
cursor.execute(
|
||||
sql,
|
||||
{
|
||||
"vehicle_limit": args.vehicles,
|
||||
"start_ts": args.start,
|
||||
"end_ts": args.end,
|
||||
},
|
||||
)
|
||||
while True:
|
||||
rows = cursor.fetchmany(args.fetch_size)
|
||||
if not rows:
|
||||
break
|
||||
for source_row in rows:
|
||||
source_rows_read += 1
|
||||
order_key = (source_row[2], source_row[0])
|
||||
if last_order_key is not None and order_key <= last_order_key:
|
||||
raise RuntimeError(
|
||||
"Source join did not return a unique increasing "
|
||||
"event_ts/internal_event_id order"
|
||||
)
|
||||
last_order_key = order_key
|
||||
transformed = secure_export.private_row(
|
||||
source_row, key, key_version
|
||||
)
|
||||
if transformed is None:
|
||||
invalid_vins_skipped += 1
|
||||
continue
|
||||
writer.writerow(transformed)
|
||||
rows_written += 1
|
||||
connection.rollback()
|
||||
except Exception:
|
||||
partial.unlink(missing_ok=True)
|
||||
partial_manifest.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
manifest_data = {
|
||||
"export_kind": "vehicle_history_development_sample",
|
||||
"query_version": QUERY_VERSION,
|
||||
"query_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(),
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"source_start_inclusive": args.start.isoformat(),
|
||||
"source_end_exclusive": args.end.isoformat(),
|
||||
"vehicle_sample_limit": args.vehicles,
|
||||
"sample_method": "page_sample_candidates_then_md5_ordered_distinct_vins",
|
||||
"sample_percent": args.sample_percent,
|
||||
"sample_seed": args.seed,
|
||||
"population_estimate_allowed": False,
|
||||
"vehicle_token_key_version": key_version,
|
||||
"vehicle_token_key_fingerprint": key_fingerprint,
|
||||
"source_rows_read": source_rows_read,
|
||||
"rows": rows_written,
|
||||
"invalid_vins_skipped": invalid_vins_skipped,
|
||||
"columns": secure_export.OUTPUT_COLUMNS,
|
||||
"compressed_file_sha256": file_sha256(partial),
|
||||
"compressed_file_bytes": partial.stat().st_size,
|
||||
"classification": "private_pseudonymized_development_only",
|
||||
"source_transaction": "read_only_repeatable_read",
|
||||
}
|
||||
partial_manifest.write_text(
|
||||
json.dumps(manifest_data, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
os.replace(partial, output)
|
||||
os.replace(partial_manifest, manifest)
|
||||
print(f"Exported {rows_written:,} rows for a development vehicle sample")
|
||||
print(f"Private output: {output}")
|
||||
print("This sample is not valid for population-rate estimates.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
414
scripts/export_inspection_batch.py
Normal file
414
scripts/export_inspection_batch.py
Normal file
@ -0,0 +1,414 @@
|
||||
"""Export one private inspection batch while replacing VINs with keyed HMACs.
|
||||
|
||||
The output is private analytical staging data, not a public-dashboard dataset.
|
||||
It intentionally omits plates, ZIPs, stations, raw JSON, and operational data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import gzip
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional, Sequence
|
||||
|
||||
import psycopg2
|
||||
|
||||
|
||||
QUERY_VERSION = "inspection_batch_v4"
|
||||
MAX_BATCH_DAYS = 32
|
||||
SECURE_SSL_MODES = {"require", "verify-ca", "verify-full"}
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_KEY_FILE = PROJECT_ROOT / ".secrets/vin_hmac.key"
|
||||
PRIVATE_DATA_ROOT = (PROJECT_ROOT / "data/private").resolve()
|
||||
VIN_PATTERN = re.compile(r"^[A-HJ-NPR-Z0-9]{17}$")
|
||||
KEY_VERSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,32}$")
|
||||
|
||||
OUTCOME_ALIASES = {
|
||||
"PASS": "pass",
|
||||
"P": "pass",
|
||||
"FAIL": "fail",
|
||||
"F": "fail",
|
||||
"REJECT": "reject",
|
||||
"ABORT": "abort",
|
||||
}
|
||||
|
||||
|
||||
def _normalized_outcome(value: object) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
return OUTCOME_ALIASES.get(value.strip().upper())
|
||||
|
||||
|
||||
def canonicalize_outcome(
|
||||
overall_result: object,
|
||||
obd_result: object,
|
||||
source_era: object,
|
||||
program_type: object = None,
|
||||
test_type: object = None,
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Return the approved label and its provenance for one inspection."""
|
||||
overall = _normalized_outcome(overall_result)
|
||||
if overall is not None:
|
||||
return overall, "overall_result"
|
||||
|
||||
proxy_eligible = (
|
||||
isinstance(source_era, str)
|
||||
and source_era.strip().lower() == "utah"
|
||||
and isinstance(program_type, str)
|
||||
and program_type.strip().lower() == "obd"
|
||||
and isinstance(test_type, str)
|
||||
and test_type.strip().upper() == "OBD"
|
||||
)
|
||||
if proxy_eligible:
|
||||
proxy = _normalized_outcome(obd_result)
|
||||
if proxy is not None:
|
||||
# This narrowly validated Utah OBD/OBD proxy is approved only for
|
||||
# binary pass/non-pass labels. Multiclass work must retain
|
||||
# overall_result labels exclusively.
|
||||
return proxy, "utah_obd_proxy"
|
||||
return None, None
|
||||
|
||||
|
||||
def _outcome_case_sql(column: str) -> str:
|
||||
cases = " ".join(
|
||||
f"WHEN '{raw}' THEN '{canonical}'"
|
||||
for raw, canonical in OUTCOME_ALIASES.items()
|
||||
)
|
||||
return f"CASE upper(btrim({column})) {cases} ELSE NULL END"
|
||||
|
||||
|
||||
_OVERALL_OUTCOME_SQL = _outcome_case_sql("s.overall_result")
|
||||
_UTAH_OBD_OUTCOME_SQL = _outcome_case_sql("s.obd_result")
|
||||
_UTAH_OBD_PROXY_PREDICATE_SQL = """lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD'"""
|
||||
|
||||
# Keep label provenance in private staging. Only Utah rows normalized to both
|
||||
# program_type=obd and test_type=OBD may use the proxy, and then only for binary
|
||||
# pass/non-pass modeling. Multiclass labels must use overall_result.
|
||||
OUTCOME_SELECT_SQL = f"""CASE
|
||||
WHEN {_OVERALL_OUTCOME_SQL} IS NOT NULL THEN {_OVERALL_OUTCOME_SQL}
|
||||
WHEN {_UTAH_OBD_PROXY_PREDICATE_SQL} THEN {_UTAH_OBD_OUTCOME_SQL}
|
||||
ELSE NULL
|
||||
END AS canonical_outcome,
|
||||
CASE
|
||||
WHEN {_OVERALL_OUTCOME_SQL} IS NOT NULL THEN 'overall_result'
|
||||
WHEN {_UTAH_OBD_PROXY_PREDICATE_SQL}
|
||||
AND {_UTAH_OBD_OUTCOME_SQL} IS NOT NULL THEN 'utah_obd_proxy'
|
||||
ELSE NULL
|
||||
END AS outcome_label_source"""
|
||||
|
||||
|
||||
EXTRACT_SQL = f"""
|
||||
SELECT
|
||||
s.id AS internal_event_id,
|
||||
s.vin,
|
||||
s.test_start AS event_ts,
|
||||
lower(btrim(s.county)) AS source_era,
|
||||
CASE
|
||||
WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake'
|
||||
ELSE lower(btrim(s.county))
|
||||
END AS public_county,
|
||||
{OUTCOME_SELECT_SQL},
|
||||
lower(nullif(btrim(s.program_type), '')) AS program_type,
|
||||
upper(nullif(btrim(s.test_type), '')) AS test_type,
|
||||
upper(nullif(btrim(v.make), '')) AS observed_make,
|
||||
upper(nullif(btrim(v.model), '')) AS observed_model,
|
||||
v.year AS observed_model_year
|
||||
FROM data.inspection_search AS s
|
||||
JOIN data.inspection_vehicle AS v USING (id)
|
||||
WHERE s.test_start >= %(start_ts)s
|
||||
AND s.test_start < %(end_ts)s
|
||||
AND s.vin IS NOT NULL
|
||||
AND length(btrim(s.vin)) = 17
|
||||
ORDER BY s.test_start, s.id
|
||||
"""
|
||||
|
||||
OUTPUT_COLUMNS = (
|
||||
"internal_event_id",
|
||||
"vehicle_token",
|
||||
"vehicle_bucket",
|
||||
"event_ts",
|
||||
"source_era",
|
||||
"public_county",
|
||||
"canonical_outcome",
|
||||
"outcome_label_source",
|
||||
"program_type",
|
||||
"test_type",
|
||||
"observed_make",
|
||||
"observed_model",
|
||||
"observed_model_year",
|
||||
)
|
||||
|
||||
|
||||
def parse_timestamp(value: str) -> datetime:
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"Invalid ISO timestamp or date: {value}"
|
||||
) from exc
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--start", required=True, type=parse_timestamp)
|
||||
parser.add_argument("--end", required=True, type=parse_timestamp)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--page-size", type=int, default=10_000)
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
parser.add_argument(
|
||||
"--allow-external-output",
|
||||
action="store_true",
|
||||
help="Allow private row-level output outside data/private (unsafe).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_args(args: argparse.Namespace) -> Path:
|
||||
if args.end <= args.start:
|
||||
raise ValueError("--end must be later than --start")
|
||||
if (args.end - args.start).total_seconds() > MAX_BATCH_DAYS * 86400:
|
||||
raise ValueError(
|
||||
f"A source batch may span at most {MAX_BATCH_DAYS} days; "
|
||||
"export longer periods as closed monthly batches."
|
||||
)
|
||||
if not 100 <= args.page_size <= 50_000:
|
||||
raise ValueError("--page-size must be between 100 and 50000")
|
||||
|
||||
output = args.output
|
||||
if output is None:
|
||||
name = f"inspection_{args.start:%Y%m%d}_{args.end:%Y%m%d}.csv.gz"
|
||||
output = PRIVATE_DATA_ROOT / "inspection_batches" / name
|
||||
output = output.resolve()
|
||||
if not args.allow_external_output and not output.is_relative_to(PRIVATE_DATA_ROOT):
|
||||
raise ValueError(
|
||||
f"Private exports must stay under {PRIVATE_DATA_ROOT}; "
|
||||
"use --allow-external-output only for controlled validation."
|
||||
)
|
||||
manifest = output.with_name(output.name + ".manifest.json")
|
||||
if (output.exists() or manifest.exists()) and not args.overwrite:
|
||||
raise FileExistsError(
|
||||
f"Refusing to overwrite {output} or its manifest; use --overwrite"
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def get_hmac_configuration() -> tuple[bytes, str, str]:
|
||||
encoded_key = os.environ.get("VIN_HASH_KEY", "").strip()
|
||||
key_file = Path(os.environ.get("VIN_HASH_KEY_FILE", DEFAULT_KEY_FILE))
|
||||
if not encoded_key and key_file.exists():
|
||||
mode = stat.S_IMODE(key_file.stat().st_mode)
|
||||
if mode & 0o077:
|
||||
raise ValueError(f"VIN HMAC key file must be mode 0600: {key_file}")
|
||||
encoded_key = key_file.read_text(encoding="ascii").strip()
|
||||
|
||||
version = os.environ.get("VIN_HASH_KEY_VERSION", "v1").strip() or "v1"
|
||||
if not KEY_VERSION_PATTERN.fullmatch(version):
|
||||
raise ValueError("VIN_HASH_KEY_VERSION contains unsupported characters")
|
||||
try:
|
||||
key = bytes.fromhex(encoded_key)
|
||||
except ValueError as exc:
|
||||
raise ValueError("VIN_HASH_KEY must be a hexadecimal value") from exc
|
||||
if len(key) != 32:
|
||||
raise ValueError(
|
||||
"VIN_HASH_KEY must encode exactly 32 random bytes (64 hex characters). "
|
||||
"See .env.example."
|
||||
)
|
||||
if len(set(key)) < 8:
|
||||
raise ValueError(
|
||||
"VIN_HASH_KEY has insufficient byte diversity; generate 32 random bytes."
|
||||
)
|
||||
fingerprint = hashlib.sha256(
|
||||
b"utah-vehicle-health/key-fingerprint/v1\0" + key
|
||||
).hexdigest()[:16]
|
||||
return key, version, fingerprint
|
||||
|
||||
|
||||
def normalize_vin(vin: str) -> Optional[str]:
|
||||
normalized = vin.strip().upper()
|
||||
if not VIN_PATTERN.fullmatch(normalized):
|
||||
return None
|
||||
if len(set(normalized)) < 4:
|
||||
return None
|
||||
if normalized in {"12345678901234567", "98765432109876543"}:
|
||||
return None
|
||||
return normalized
|
||||
|
||||
|
||||
def vehicle_token(vin: str, key: bytes, key_version: str) -> str:
|
||||
message = (
|
||||
f"utah-vehicle-health/{key_version}/vin\0".encode("ascii")
|
||||
+ vin.encode("ascii")
|
||||
)
|
||||
return hmac.new(key, message, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def private_row(
|
||||
row: Sequence[object], key: bytes, key_version: str
|
||||
) -> Optional[tuple[object, ...]]:
|
||||
internal_event_id, vin, *remaining = row
|
||||
if not isinstance(vin, str):
|
||||
raise ValueError("The source returned a non-text VIN")
|
||||
normalized = normalize_vin(vin)
|
||||
if normalized is None:
|
||||
return None
|
||||
token = vehicle_token(normalized, key, key_version)
|
||||
bucket = int(token[:8], 16) % 100
|
||||
return (internal_event_id, token, bucket, *remaining)
|
||||
|
||||
|
||||
def batches(
|
||||
connection: "psycopg2.extensions.connection",
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
page_size: int,
|
||||
) -> Iterable[list[tuple[object, ...]]]:
|
||||
with connection.cursor(name="uvh_inspection_batch") as cursor:
|
||||
cursor.itersize = page_size
|
||||
cursor.execute(EXTRACT_SQL, {"start_ts": start, "end_ts": end})
|
||||
while True:
|
||||
rows = cursor.fetchmany(page_size)
|
||||
if not rows:
|
||||
return
|
||||
yield rows
|
||||
|
||||
|
||||
def connect_read_only() -> "psycopg2.extensions.connection":
|
||||
ssl_mode = os.environ.get("PGSSLMODE", "").lower()
|
||||
if ssl_mode not in SECURE_SSL_MODES:
|
||||
raise ValueError("PGSSLMODE must require TLS before exporting data")
|
||||
|
||||
connection = psycopg2.connect(
|
||||
application_name="utah_vehicle_health_batch_export",
|
||||
connect_timeout=10,
|
||||
options=(
|
||||
"-c default_transaction_read_only=on "
|
||||
"-c statement_timeout=120000 "
|
||||
"-c lock_timeout=3000 "
|
||||
"-c idle_in_transaction_session_timeout=60000"
|
||||
),
|
||||
)
|
||||
connection.set_session(
|
||||
readonly=True,
|
||||
isolation_level="REPEATABLE READ",
|
||||
autocommit=False,
|
||||
)
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
current_setting('transaction_read_only'),
|
||||
ssl
|
||||
FROM pg_stat_ssl
|
||||
WHERE pid = pg_backend_pid()
|
||||
"""
|
||||
)
|
||||
read_only, ssl = cursor.fetchone()
|
||||
if read_only != "on" or not ssl:
|
||||
connection.close()
|
||||
raise RuntimeError("The database session is not read-only over TLS")
|
||||
return connection
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
os.umask(0o077)
|
||||
try:
|
||||
output = validate_args(args)
|
||||
key, key_version, key_fingerprint = get_hmac_configuration()
|
||||
except (FileExistsError, ValueError) as exc:
|
||||
print(f"Configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = output.with_name(output.name + ".partial")
|
||||
manifest = output.with_name(output.name + ".manifest.json")
|
||||
partial_manifest = manifest.with_name(manifest.name + ".partial")
|
||||
|
||||
try:
|
||||
connection = connect_read_only()
|
||||
except (ValueError, RuntimeError, psycopg2.Error) as exc:
|
||||
print(f"Secure read-only connection failed: {type(exc).__name__}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
source_rows_read = 0
|
||||
rows_written = 0
|
||||
invalid_vins_skipped = 0
|
||||
last_order_key: Optional[tuple[object, object]] = None
|
||||
try:
|
||||
with gzip.open(partial, "wt", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.writer(handle)
|
||||
writer.writerow(OUTPUT_COLUMNS)
|
||||
for source_rows in batches(
|
||||
connection, args.start, args.end, args.page_size
|
||||
):
|
||||
for source_row in source_rows:
|
||||
source_rows_read += 1
|
||||
order_key = (source_row[2], source_row[0])
|
||||
if last_order_key is not None and order_key <= last_order_key:
|
||||
raise RuntimeError(
|
||||
"Source join did not return a unique increasing "
|
||||
"event_ts/internal_event_id order"
|
||||
)
|
||||
last_order_key = order_key
|
||||
transformed = private_row(source_row, key, key_version)
|
||||
if transformed is None:
|
||||
invalid_vins_skipped += 1
|
||||
continue
|
||||
writer.writerow(transformed)
|
||||
rows_written += 1
|
||||
connection.rollback()
|
||||
except Exception:
|
||||
partial.unlink(missing_ok=True)
|
||||
partial_manifest.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
digest = hashlib.sha256()
|
||||
with partial.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
|
||||
manifest_data = {
|
||||
"export_kind": "bounded_batch",
|
||||
"query_version": QUERY_VERSION,
|
||||
"query_sha256": hashlib.sha256(EXTRACT_SQL.encode("utf-8")).hexdigest(),
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"source_start_inclusive": args.start.isoformat(),
|
||||
"source_end_exclusive": args.end.isoformat(),
|
||||
"vehicle_token_key_version": key_version,
|
||||
"vehicle_token_key_fingerprint": key_fingerprint,
|
||||
"source_rows_read": source_rows_read,
|
||||
"rows": rows_written,
|
||||
"invalid_vins_skipped": invalid_vins_skipped,
|
||||
"columns": OUTPUT_COLUMNS,
|
||||
"compressed_file_sha256": digest.hexdigest(),
|
||||
"compressed_file_bytes": partial.stat().st_size,
|
||||
"classification": "private_pseudonymized_analytical_staging",
|
||||
"source_transaction": "read_only_repeatable_read",
|
||||
}
|
||||
partial_manifest.write_text(
|
||||
json.dumps(manifest_data, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
os.replace(partial, output)
|
||||
os.replace(partial_manifest, manifest)
|
||||
print(f"Exported {rows_written:,} private pseudonymized rows to {output}")
|
||||
if invalid_vins_skipped:
|
||||
print(f"Skipped {invalid_vins_skipped:,} invalid/placeholder VIN rows")
|
||||
print(f"Wrote manifest to {manifest}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
71
scripts/run_feasibility_check.py
Normal file
71
scripts/run_feasibility_check.py
Normal file
@ -0,0 +1,71 @@
|
||||
"""Run the project's fixed aggregate-only feasibility query safely.
|
||||
|
||||
This runner accepts no arbitrary SQL. It executes the reviewed query inside the
|
||||
same TLS-protected, read-only transaction used by the private exporters and
|
||||
prints only its aggregate result table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg2
|
||||
|
||||
import export_inspection_batch as secure_export
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
QUERY_PATH = PROJECT_ROOT / "sql/10_episode_cohort_feasibility.sql"
|
||||
|
||||
|
||||
def reviewed_query() -> str:
|
||||
sql = QUERY_PATH.read_text(encoding="utf-8")
|
||||
begin = "BEGIN TRANSACTION READ ONLY;"
|
||||
rollback = "ROLLBACK;"
|
||||
if sql.count(begin) != 1 or sql.count(rollback) != 1:
|
||||
raise RuntimeError("The reviewed feasibility query wrappers changed")
|
||||
return sql.replace(begin, "", 1).rsplit(rollback, 1)[0].strip()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
connection = secure_export.connect_read_only()
|
||||
except (ValueError, RuntimeError, psycopg2.Error) as exc:
|
||||
print(
|
||||
f"Secure read-only connection failed: {type(exc).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(reviewed_query())
|
||||
columns = [description.name for description in cursor.description]
|
||||
rows = cursor.fetchall()
|
||||
connection.rollback()
|
||||
except psycopg2.Error as exc:
|
||||
connection.rollback()
|
||||
print(
|
||||
f"Aggregate feasibility query failed: "
|
||||
f"{exc.diag.message_primary or type(exc).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
widths = [len(column) for column in columns]
|
||||
rendered = [["" if value is None else str(value) for value in row] for row in rows]
|
||||
for row in rendered:
|
||||
widths = [max(width, len(value)) for width, value in zip(widths, row)]
|
||||
|
||||
print(" ".join(value.ljust(width) for value, width in zip(columns, widths)))
|
||||
print(" ".join("-" * width for width in widths))
|
||||
for row in rendered:
|
||||
print(" ".join(value.ljust(width) for value, width in zip(row, widths)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
65
scripts/run_outcome_mapping_audit.py
Normal file
65
scripts/run_outcome_mapping_audit.py
Normal file
@ -0,0 +1,65 @@
|
||||
"""Run the fixed aggregate-only source outcome-mapping audit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg2
|
||||
|
||||
import export_inspection_batch as secure_export
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
QUERY_PATH = PROJECT_ROOT / "sql/11_outcome_mapping_audit.sql"
|
||||
|
||||
|
||||
def reviewed_query() -> str:
|
||||
sql = QUERY_PATH.read_text(encoding="utf-8")
|
||||
begin = "BEGIN TRANSACTION READ ONLY;"
|
||||
rollback = "ROLLBACK;"
|
||||
if sql.count(begin) != 1 or sql.count(rollback) != 1:
|
||||
raise RuntimeError("The reviewed outcome audit wrappers changed")
|
||||
return sql.replace(begin, "", 1).rsplit(rollback, 1)[0].strip()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
connection = secure_export.connect_read_only()
|
||||
except (ValueError, RuntimeError, psycopg2.Error) as exc:
|
||||
print(
|
||||
f"Secure read-only connection failed: {type(exc).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(reviewed_query())
|
||||
columns = [description.name for description in cursor.description]
|
||||
rows = cursor.fetchall()
|
||||
connection.rollback()
|
||||
except psycopg2.Error as exc:
|
||||
connection.rollback()
|
||||
print(
|
||||
"Aggregate outcome audit failed: "
|
||||
f"{exc.diag.message_primary or type(exc).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
widths = [len(column) for column in columns]
|
||||
rendered = [["" if value is None else str(value) for value in row] for row in rows]
|
||||
for row in rendered:
|
||||
widths = [max(width, len(value)) for width, value in zip(widths, row)]
|
||||
print(" ".join(value.ljust(width) for value, width in zip(columns, widths)))
|
||||
print(" ".join("-" * width for width in widths))
|
||||
for row in rendered:
|
||||
print(" ".join(value.ljust(width) for value, width in zip(row, widths)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1469
scripts/train_baselines.py
Normal file
1469
scripts/train_baselines.py
Normal file
File diff suppressed because it is too large
Load Diff
799
scripts/train_tree_model.py
Normal file
799
scripts/train_tree_model.py
Normal file
@ -0,0 +1,799 @@
|
||||
"""Train a leakage-safe nonlinear candidate from the private episode mart.
|
||||
|
||||
The model is fitted only on non-audit 2016-2022 episodes, selected on 2023,
|
||||
and probability-calibrated on 2024. The locked 2025 partition is evaluated
|
||||
only when ``--evaluate-locked`` is supplied. No row-level predictions are
|
||||
written.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
try:
|
||||
from scripts import train_baselines as baselines
|
||||
except ModuleNotFoundError: # Direct execution places scripts/ on sys.path.
|
||||
import train_baselines as baselines
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODEL_VERSION = "hist_gradient_boosting_v1"
|
||||
DEFAULT_OUTPUT = PROJECT_ROOT / "artifacts/private/tree" / MODEL_VERSION
|
||||
|
||||
NUMERIC_FEATURES = baselines.NUMERIC_FEATURES
|
||||
CATEGORICAL_FEATURES = baselines.CATEGORICAL_FEATURES
|
||||
|
||||
MISSING_CATEGORY_CODE = 0
|
||||
RARE_CATEGORY_CODE = 1
|
||||
UNKNOWN_CATEGORY_CODE = 2
|
||||
FIRST_KNOWN_CATEGORY_CODE = 3
|
||||
MAX_HISTOGRAM_BINS = 255
|
||||
|
||||
# These columns are validated by the baseline mart loader but are deliberately
|
||||
# unavailable to the nonlinear feature encoder.
|
||||
EXCLUDED_FROM_PREDICTORS = (
|
||||
"vehicle_token",
|
||||
"vehicle_bucket",
|
||||
"is_vin_audit",
|
||||
"episode_number",
|
||||
"episode_start",
|
||||
"first_outcome",
|
||||
"target_nonpass",
|
||||
"eligible_returning_target",
|
||||
"temporal_partition",
|
||||
"source_era",
|
||||
"target_outcome_label_source",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TreeCandidate:
|
||||
learning_rate: float
|
||||
max_leaf_nodes: int
|
||||
l2_regularization: float
|
||||
|
||||
def artifact_state(self) -> Dict[str, object]:
|
||||
return {
|
||||
"learning_rate": self.learning_rate,
|
||||
"max_leaf_nodes": self.max_leaf_nodes,
|
||||
"l2_regularization": self.l2_regularization,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TreeFeatureEncoder:
|
||||
"""Train-only median imputation and bounded ordinal category encoding."""
|
||||
|
||||
min_category_count: int
|
||||
max_categories: int
|
||||
numeric_medians: Dict[str, float]
|
||||
seen_categories: Dict[str, set]
|
||||
known_category_codes: Dict[str, Dict[str, int]]
|
||||
feature_names: Tuple[str, ...]
|
||||
categorical_mask: Tuple[bool, ...]
|
||||
|
||||
@classmethod
|
||||
def fit(
|
||||
cls,
|
||||
rows: Sequence[baselines.EpisodeRow],
|
||||
min_category_count: int,
|
||||
max_categories: int,
|
||||
) -> "TreeFeatureEncoder":
|
||||
if not rows:
|
||||
raise baselines.DataValidationError(
|
||||
"Cannot fit tree preprocessing on zero rows"
|
||||
)
|
||||
if min_category_count < 1:
|
||||
raise baselines.DataValidationError(
|
||||
"min_category_count must be at least 1"
|
||||
)
|
||||
if not FIRST_KNOWN_CATEGORY_CODE + 1 <= max_categories <= MAX_HISTOGRAM_BINS:
|
||||
raise baselines.DataValidationError(
|
||||
"max_categories must be between {} and {}".format(
|
||||
FIRST_KNOWN_CATEGORY_CODE + 1, MAX_HISTOGRAM_BINS
|
||||
)
|
||||
)
|
||||
|
||||
numeric_medians: Dict[str, float] = {}
|
||||
for name in NUMERIC_FEATURES:
|
||||
observed = [
|
||||
row.numeric[name]
|
||||
for row in rows
|
||||
if row.numeric[name] is not None
|
||||
]
|
||||
if not observed:
|
||||
raise baselines.DataValidationError(
|
||||
"Training data has no observed values for numeric feature " + name
|
||||
)
|
||||
median = float(statistics.median(observed))
|
||||
if not math.isfinite(median):
|
||||
raise baselines.DataValidationError(
|
||||
"Training median is non-finite for numeric feature " + name
|
||||
)
|
||||
numeric_medians[name] = median
|
||||
|
||||
seen_categories: Dict[str, set] = {}
|
||||
known_category_codes: Dict[str, Dict[str, int]] = {}
|
||||
category_capacity = max_categories - FIRST_KNOWN_CATEGORY_CODE
|
||||
for name in CATEGORICAL_FEATURES:
|
||||
observed_values = [
|
||||
row.categorical[name]
|
||||
for row in rows
|
||||
if row.categorical[name] is not None
|
||||
]
|
||||
counts = Counter(observed_values)
|
||||
seen_categories[name] = set(counts)
|
||||
candidates = [
|
||||
(value, count)
|
||||
for value, count in counts.items()
|
||||
if count >= min_category_count
|
||||
]
|
||||
# Frequency first, lexical second makes capping deterministic.
|
||||
candidates.sort(key=lambda item: (-item[1], item[0]))
|
||||
kept = candidates[:category_capacity]
|
||||
known_category_codes[name] = {
|
||||
value: FIRST_KNOWN_CATEGORY_CODE + index
|
||||
for index, (value, _count) in enumerate(kept)
|
||||
}
|
||||
|
||||
feature_names = tuple(
|
||||
list(NUMERIC_FEATURES)
|
||||
+ ["missing__" + name for name in NUMERIC_FEATURES]
|
||||
+ list(CATEGORICAL_FEATURES)
|
||||
)
|
||||
categorical_mask = tuple(
|
||||
[False] * (2 * len(NUMERIC_FEATURES))
|
||||
+ [True] * len(CATEGORICAL_FEATURES)
|
||||
)
|
||||
return cls(
|
||||
min_category_count=min_category_count,
|
||||
max_categories=max_categories,
|
||||
numeric_medians=numeric_medians,
|
||||
seen_categories=seen_categories,
|
||||
known_category_codes=known_category_codes,
|
||||
feature_names=feature_names,
|
||||
categorical_mask=categorical_mask,
|
||||
)
|
||||
|
||||
def category_code(self, feature: str, value: Optional[str]) -> int:
|
||||
if value is None:
|
||||
return MISSING_CATEGORY_CODE
|
||||
known = self.known_category_codes[feature]
|
||||
if value in known:
|
||||
return known[value]
|
||||
if value in self.seen_categories[feature]:
|
||||
return RARE_CATEGORY_CODE
|
||||
return UNKNOWN_CATEGORY_CODE
|
||||
|
||||
def transform(self, rows: Sequence[baselines.EpisodeRow], np_module: object) -> object:
|
||||
matrix = np_module.empty((len(rows), len(self.feature_names)), dtype=np_module.float64)
|
||||
for row_index, row in enumerate(rows):
|
||||
column = 0
|
||||
for name in NUMERIC_FEATURES:
|
||||
value = row.numeric[name]
|
||||
matrix[row_index, column] = (
|
||||
self.numeric_medians[name] if value is None else float(value)
|
||||
)
|
||||
column += 1
|
||||
for name in NUMERIC_FEATURES:
|
||||
matrix[row_index, column] = 1.0 if row.numeric[name] is None else 0.0
|
||||
column += 1
|
||||
for name in CATEGORICAL_FEATURES:
|
||||
matrix[row_index, column] = float(
|
||||
self.category_code(name, row.categorical[name])
|
||||
)
|
||||
column += 1
|
||||
if matrix.size and not bool(np_module.isfinite(matrix).all()):
|
||||
raise baselines.DataValidationError(
|
||||
"Tree feature matrix contains non-finite values"
|
||||
)
|
||||
return matrix
|
||||
|
||||
def artifact_state(self) -> Dict[str, object]:
|
||||
return {
|
||||
"min_category_count": self.min_category_count,
|
||||
"max_categories": self.max_categories,
|
||||
"reserved_category_codes": {
|
||||
"missing": MISSING_CATEGORY_CODE,
|
||||
"rare_seen_in_training": RARE_CATEGORY_CODE,
|
||||
"unknown_after_training": UNKNOWN_CATEGORY_CODE,
|
||||
"first_known": FIRST_KNOWN_CATEGORY_CODE,
|
||||
},
|
||||
"numeric_medians": self.numeric_medians,
|
||||
"seen_categories": {
|
||||
name: sorted(values) for name, values in self.seen_categories.items()
|
||||
},
|
||||
"known_category_codes": self.known_category_codes,
|
||||
"feature_names": self.feature_names,
|
||||
"categorical_mask": self.categorical_mask,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainedTreeModel:
|
||||
encoder: TreeFeatureEncoder
|
||||
model: object
|
||||
platt_model: object
|
||||
selected_candidate: TreeCandidate
|
||||
model_n_iter: int
|
||||
platt_n_iter: int
|
||||
max_iter: int
|
||||
calibration_max_iter: int
|
||||
tuning_results: List[Dict[str, object]]
|
||||
|
||||
|
||||
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mart", required=True, type=Path)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--seed", type=int, default=20260715)
|
||||
parser.add_argument("--min-category-count", type=int, default=100)
|
||||
parser.add_argument("--max-categories", type=int, default=128)
|
||||
parser.add_argument("--learning-rates", default="0.05,0.1")
|
||||
parser.add_argument("--max-leaf-nodes", default="15,31")
|
||||
parser.add_argument("--l2-grid", default="1.0")
|
||||
parser.add_argument("--min-samples-leaf", type=int, default=20)
|
||||
parser.add_argument("--max-iter", type=int, default=300)
|
||||
parser.add_argument("--n-iter-no-change", type=int, default=20)
|
||||
parser.add_argument("--calibration-max-iter", type=int, default=5000)
|
||||
parser.add_argument("--calibration-bins", type=int, default=10)
|
||||
parser.add_argument("--evaluate-locked", action="store_true")
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def _parse_float_grid(value: str, label: str, allow_zero: bool) -> List[float]:
|
||||
try:
|
||||
values = [float(part.strip()) for part in value.split(",") if part.strip()]
|
||||
except ValueError as exc:
|
||||
raise baselines.DataValidationError(label + " must contain finite numbers") from exc
|
||||
minimum_ok = (lambda item: item >= 0.0) if allow_zero else (lambda item: item > 0.0)
|
||||
if not values or any(not math.isfinite(item) or not minimum_ok(item) for item in values):
|
||||
raise baselines.DataValidationError(label + " contains an invalid value")
|
||||
return sorted(set(values))
|
||||
|
||||
|
||||
def _parse_int_grid(value: str, label: str, minimum: int) -> List[int]:
|
||||
try:
|
||||
values = [int(part.strip()) for part in value.split(",") if part.strip()]
|
||||
except ValueError as exc:
|
||||
raise baselines.DataValidationError(label + " must contain integers") from exc
|
||||
if not values or any(item < minimum for item in values):
|
||||
raise baselines.DataValidationError(label + " contains an invalid value")
|
||||
return sorted(set(values))
|
||||
|
||||
|
||||
def candidate_grid(
|
||||
learning_rates: Sequence[float],
|
||||
max_leaf_nodes: Sequence[int],
|
||||
l2_values: Sequence[float],
|
||||
) -> List[TreeCandidate]:
|
||||
return [
|
||||
TreeCandidate(rate, leaves, l2)
|
||||
for rate in sorted(set(learning_rates))
|
||||
for leaves in sorted(set(max_leaf_nodes))
|
||||
for l2 in sorted(set(l2_values))
|
||||
]
|
||||
|
||||
|
||||
def require_tree_dependencies() -> Tuple[object, object, object, object, object, object]:
|
||||
# joblib's macOS physical-core probe emits a UserWarning on this managed
|
||||
# host. Preserve an explicit user setting; otherwise give it the already
|
||||
# available logical count so warning-as-error validation remains usable.
|
||||
os.environ.setdefault("LOKY_MAX_CPU_COUNT", "1")
|
||||
try:
|
||||
import joblib
|
||||
import numpy as np
|
||||
import sklearn
|
||||
from sklearn.ensemble import HistGradientBoostingClassifier
|
||||
from sklearn.exceptions import ConvergenceWarning
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
except ImportError as exc:
|
||||
raise baselines.DependencyError(
|
||||
"Tree training requires numpy, scikit-learn, and joblib"
|
||||
) from exc
|
||||
return (
|
||||
np,
|
||||
sklearn,
|
||||
joblib,
|
||||
HistGradientBoostingClassifier,
|
||||
LogisticRegression,
|
||||
ConvergenceWarning,
|
||||
)
|
||||
|
||||
|
||||
def _non_audit(rows: Sequence[baselines.EpisodeRow]) -> List[baselines.EpisodeRow]:
|
||||
return [row for row in rows if not row.audit_vehicle]
|
||||
|
||||
|
||||
def _targets(rows: Sequence[baselines.EpisodeRow]) -> List[int]:
|
||||
return [row.target for row in rows]
|
||||
|
||||
|
||||
def _scores_finite(model: object, np_module: object) -> bool:
|
||||
for attribute in ("train_score_", "validation_score_"):
|
||||
values = getattr(model, attribute, None)
|
||||
if values is None:
|
||||
continue
|
||||
array = np_module.asarray(values)
|
||||
if array.size and not bool(np_module.isfinite(array).all()):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _hist_converged(
|
||||
model: object,
|
||||
convergence_messages: Sequence[str],
|
||||
max_iter: int,
|
||||
np_module: object,
|
||||
) -> Tuple[bool, Optional[int]]:
|
||||
value = getattr(model, "n_iter_", None)
|
||||
if value is None:
|
||||
return False, None
|
||||
try:
|
||||
n_iter = int(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return False, None
|
||||
converged = (
|
||||
not convergence_messages
|
||||
and 0 < n_iter < max_iter
|
||||
and _scores_finite(model, np_module)
|
||||
)
|
||||
return converged, n_iter
|
||||
|
||||
|
||||
def _decision_scores(model: object, features: object, np_module: object) -> object:
|
||||
scores = np_module.asarray(model.decision_function(features), dtype=np_module.float64)
|
||||
scores = scores.reshape(-1, 1)
|
||||
if scores.size == 0 or not bool(np_module.isfinite(scores).all()):
|
||||
raise baselines.DataValidationError(
|
||||
"Tree model produced non-finite decision scores"
|
||||
)
|
||||
return scores
|
||||
|
||||
|
||||
def train_tree_model(
|
||||
mart: baselines.MartData,
|
||||
candidates: Sequence[TreeCandidate],
|
||||
min_category_count: int,
|
||||
max_categories: int,
|
||||
min_samples_leaf: int,
|
||||
max_iter: int,
|
||||
n_iter_no_change: int,
|
||||
calibration_max_iter: int,
|
||||
seed: int,
|
||||
) -> TrainedTreeModel:
|
||||
(
|
||||
np,
|
||||
_sklearn,
|
||||
_joblib,
|
||||
hist_class,
|
||||
logistic_class,
|
||||
convergence_warning_class,
|
||||
) = require_tree_dependencies()
|
||||
if not candidates:
|
||||
raise baselines.DataValidationError("The tree tuning grid is empty")
|
||||
if min_samples_leaf < 1 or max_iter < 2 or n_iter_no_change < 1:
|
||||
raise baselines.DataValidationError("Invalid tree iteration/leaf configuration")
|
||||
if calibration_max_iter < 2:
|
||||
raise baselines.DataValidationError("calibration_max_iter must be at least 2")
|
||||
|
||||
train_rows = _non_audit(mart.rows_by_partition["train"])
|
||||
tune_rows = _non_audit(mart.rows_by_partition["tune"])
|
||||
calibrate_rows = _non_audit(mart.rows_by_partition["calibrate"])
|
||||
baselines._require_two_classes(train_rows, "Tree training")
|
||||
baselines._require_two_classes(tune_rows, "Tree tuning")
|
||||
baselines._require_two_classes(calibrate_rows, "Tree calibration")
|
||||
|
||||
encoder = TreeFeatureEncoder.fit(
|
||||
train_rows,
|
||||
min_category_count=min_category_count,
|
||||
max_categories=max_categories,
|
||||
)
|
||||
x_train = encoder.transform(train_rows, np)
|
||||
x_tune = encoder.transform(tune_rows, np)
|
||||
x_calibrate = encoder.transform(calibrate_rows, np)
|
||||
train_targets = np.asarray(_targets(train_rows), dtype=np.int8)
|
||||
tune_targets = _targets(tune_rows)
|
||||
calibrate_targets = np.asarray(_targets(calibrate_rows), dtype=np.int8)
|
||||
|
||||
selectable: List[Tuple[float, float, float, int, float, object, int, TreeCandidate]] = []
|
||||
tuning_results: List[Dict[str, object]] = []
|
||||
for candidate in candidates:
|
||||
model = hist_class(
|
||||
loss="log_loss",
|
||||
learning_rate=candidate.learning_rate,
|
||||
max_iter=max_iter,
|
||||
max_leaf_nodes=candidate.max_leaf_nodes,
|
||||
min_samples_leaf=min_samples_leaf,
|
||||
l2_regularization=candidate.l2_regularization,
|
||||
max_bins=MAX_HISTOGRAM_BINS,
|
||||
categorical_features=list(encoder.categorical_mask),
|
||||
early_stopping=True,
|
||||
scoring="loss",
|
||||
validation_fraction=0.10,
|
||||
n_iter_no_change=n_iter_no_change,
|
||||
tol=1e-7,
|
||||
random_state=seed,
|
||||
class_weight=None,
|
||||
)
|
||||
result: Dict[str, object] = {
|
||||
**candidate.artifact_state(),
|
||||
"converged": False,
|
||||
"n_iter": None,
|
||||
"convergence_warning": None,
|
||||
"finite_scores": False,
|
||||
"finite_train_probabilities": False,
|
||||
"finite_tune_probabilities": False,
|
||||
"eligible_for_selection": False,
|
||||
"brier": None,
|
||||
"average_precision": None,
|
||||
}
|
||||
convergence_messages = baselines.fit_with_convergence_capture(
|
||||
model, x_train, train_targets, convergence_warning_class
|
||||
)
|
||||
converged, n_iter = _hist_converged(
|
||||
model, convergence_messages, max_iter, np
|
||||
)
|
||||
result["converged"] = converged
|
||||
result["n_iter"] = n_iter
|
||||
result["convergence_warning"] = (
|
||||
" | ".join(convergence_messages) if convergence_messages else None
|
||||
)
|
||||
result["finite_scores"] = _scores_finite(model, np)
|
||||
|
||||
train_probabilities = model.predict_proba(x_train)[:, 1]
|
||||
tune_probabilities = model.predict_proba(x_tune)[:, 1]
|
||||
finite_train = baselines.probability_array_finite(train_probabilities, np)
|
||||
finite_tune = baselines.probability_array_finite(tune_probabilities, np)
|
||||
result["finite_train_probabilities"] = finite_train
|
||||
result["finite_tune_probabilities"] = finite_tune
|
||||
eligible = converged and finite_train and finite_tune
|
||||
result["eligible_for_selection"] = eligible
|
||||
if eligible and n_iter is not None:
|
||||
metrics = baselines.binary_metrics(tune_targets, tune_probabilities.tolist())
|
||||
brier = float(metrics["brier"])
|
||||
average_precision = float(metrics["average_precision"])
|
||||
result["brier"] = brier
|
||||
result["average_precision"] = average_precision
|
||||
selectable.append(
|
||||
(
|
||||
brier,
|
||||
-average_precision,
|
||||
candidate.learning_rate,
|
||||
candidate.max_leaf_nodes,
|
||||
candidate.l2_regularization,
|
||||
model,
|
||||
n_iter,
|
||||
candidate,
|
||||
)
|
||||
)
|
||||
tuning_results.append(result)
|
||||
|
||||
if not selectable:
|
||||
raise baselines.DataValidationError(
|
||||
"No tree candidate converged with finite train/tune probabilities; "
|
||||
"increase --max-iter or inspect the feature contract"
|
||||
)
|
||||
(
|
||||
_brier,
|
||||
_negative_ap,
|
||||
_rate,
|
||||
_leaves,
|
||||
_l2,
|
||||
selected_model,
|
||||
selected_n_iter,
|
||||
selected_candidate,
|
||||
) = min(selectable, key=lambda item: item[:5])
|
||||
|
||||
calibration_scores = _decision_scores(selected_model, x_calibrate, np)
|
||||
platt_model = logistic_class(
|
||||
C=baselines.PLATT_CALIBRATION_CONFIG["c"],
|
||||
penalty=baselines.PLATT_CALIBRATION_CONFIG["penalty"],
|
||||
solver=baselines.PLATT_CALIBRATION_CONFIG["solver"],
|
||||
class_weight=None,
|
||||
max_iter=calibration_max_iter,
|
||||
random_state=seed,
|
||||
)
|
||||
platt_warnings = baselines.fit_with_convergence_capture(
|
||||
platt_model,
|
||||
calibration_scores,
|
||||
calibrate_targets,
|
||||
convergence_warning_class,
|
||||
)
|
||||
platt_n_iter = baselines.model_n_iter(platt_model, np)
|
||||
if (
|
||||
platt_warnings
|
||||
or platt_n_iter is None
|
||||
or platt_n_iter >= calibration_max_iter
|
||||
or not baselines.model_parameters_finite(platt_model, np)
|
||||
):
|
||||
detail = " | ".join(platt_warnings) if platt_warnings else "finite/convergence check"
|
||||
raise baselines.DataValidationError(
|
||||
"Tree Platt calibration did not converge: " + detail
|
||||
)
|
||||
calibrated = platt_model.predict_proba(calibration_scores)[:, 1]
|
||||
if not baselines.probability_array_finite(calibrated, np):
|
||||
raise baselines.DataValidationError(
|
||||
"Tree Platt calibration produced non-finite probabilities"
|
||||
)
|
||||
|
||||
return TrainedTreeModel(
|
||||
encoder=encoder,
|
||||
model=selected_model,
|
||||
platt_model=platt_model,
|
||||
selected_candidate=selected_candidate,
|
||||
model_n_iter=selected_n_iter,
|
||||
platt_n_iter=int(platt_n_iter),
|
||||
max_iter=max_iter,
|
||||
calibration_max_iter=calibration_max_iter,
|
||||
tuning_results=tuning_results,
|
||||
)
|
||||
|
||||
|
||||
def tree_probabilities(
|
||||
trained: TrainedTreeModel,
|
||||
rows: Sequence[baselines.EpisodeRow],
|
||||
calibrated: bool,
|
||||
) -> List[float]:
|
||||
np, _sklearn, _joblib, _hist, _logistic, _warning = require_tree_dependencies()
|
||||
matrix = trained.encoder.transform(rows, np)
|
||||
if calibrated:
|
||||
scores = _decision_scores(trained.model, matrix, np)
|
||||
probabilities = trained.platt_model.predict_proba(scores)[:, 1]
|
||||
else:
|
||||
probabilities = trained.model.predict_proba(matrix)[:, 1]
|
||||
if not baselines.probability_array_finite(probabilities, np):
|
||||
raise baselines.DataValidationError(
|
||||
"Tree model produced non-finite probabilities"
|
||||
)
|
||||
return probabilities.tolist()
|
||||
|
||||
|
||||
def evaluate_tree_model(
|
||||
mart: baselines.MartData,
|
||||
trained: TrainedTreeModel,
|
||||
evaluate_locked: bool,
|
||||
bin_count: int,
|
||||
) -> Tuple[List[Dict[str, object]], List[Dict[str, object]]]:
|
||||
metric_rows: List[Dict[str, object]] = []
|
||||
calibration_rows: List[Dict[str, object]] = []
|
||||
partitions = ["train", "tune", "calibrate"]
|
||||
if evaluate_locked:
|
||||
partitions.append("locked_test")
|
||||
|
||||
for partition in partitions:
|
||||
source_rows = mart.rows_by_partition[partition]
|
||||
cohorts = baselines.evaluation_cohorts(
|
||||
source_rows, include_audit_breakout=(partition == "locked_test")
|
||||
)
|
||||
for cohort_name, rows in cohorts:
|
||||
if not rows:
|
||||
continue
|
||||
targets = _targets(rows)
|
||||
predictions = [
|
||||
(
|
||||
"hist_gradient_boosting_raw",
|
||||
tree_probabilities(trained, rows, calibrated=False),
|
||||
)
|
||||
]
|
||||
if partition in {"calibrate", "locked_test"}:
|
||||
predictions.append(
|
||||
(
|
||||
"hist_gradient_boosting_platt",
|
||||
tree_probabilities(trained, rows, calibrated=True),
|
||||
)
|
||||
)
|
||||
for model_name, probabilities in predictions:
|
||||
metrics = baselines.binary_metrics(targets, probabilities)
|
||||
metric_rows.append(
|
||||
{
|
||||
"model": model_name,
|
||||
"partition": partition,
|
||||
"cohort": cohort_name,
|
||||
"episodes": len(rows),
|
||||
"vehicles": len({row.vehicle_token for row in rows}),
|
||||
"nonpass": sum(targets),
|
||||
**metrics,
|
||||
}
|
||||
)
|
||||
for values in baselines.calibration_bins(
|
||||
targets, probabilities, bin_count
|
||||
):
|
||||
calibration_rows.append(
|
||||
{
|
||||
"model": model_name,
|
||||
"partition": partition,
|
||||
"cohort": cohort_name,
|
||||
**values,
|
||||
}
|
||||
)
|
||||
return metric_rows, calibration_rows
|
||||
|
||||
|
||||
def _input_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_artifacts(
|
||||
output_dir: Path,
|
||||
mart_path: Path,
|
||||
mart: baselines.MartData,
|
||||
trained: TrainedTreeModel,
|
||||
metric_rows: Sequence[Mapping[str, object]],
|
||||
calibration_rows: Sequence[Mapping[str, object]],
|
||||
evaluate_locked: bool,
|
||||
seed: int,
|
||||
overwrite: bool,
|
||||
sklearn_version: str,
|
||||
) -> None:
|
||||
_np, _sklearn, joblib, _hist, _logistic, _warning = require_tree_dependencies()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
model_path = output_dir / "model.joblib"
|
||||
metrics_path = output_dir / "metrics.json"
|
||||
calibration_path = output_dir / "calibration_bins.csv"
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
expected = (model_path, metrics_path, calibration_path, manifest_path)
|
||||
existing = [path for path in expected if path.exists()]
|
||||
if existing and not overwrite:
|
||||
raise baselines.DataValidationError(
|
||||
"Refusing to overwrite existing tree artifacts: "
|
||||
+ ", ".join(path.name for path in existing)
|
||||
)
|
||||
|
||||
partial_model = model_path.with_name(model_path.name + ".partial")
|
||||
joblib.dump(
|
||||
{
|
||||
"model_version": MODEL_VERSION,
|
||||
"target_contract": baselines.TARGET_CONTRACT,
|
||||
"numeric_features": NUMERIC_FEATURES,
|
||||
"categorical_features": CATEGORICAL_FEATURES,
|
||||
"excluded_from_predictors": EXCLUDED_FROM_PREDICTORS,
|
||||
"encoder": trained.encoder.artifact_state(),
|
||||
"selected_candidate": trained.selected_candidate.artifact_state(),
|
||||
"hist_gradient_boosting_model": trained.model,
|
||||
"platt_model": trained.platt_model,
|
||||
"platt_calibration_config": baselines.PLATT_CALIBRATION_CONFIG,
|
||||
},
|
||||
partial_model,
|
||||
)
|
||||
os.replace(partial_model, model_path)
|
||||
baselines._atomic_text(
|
||||
metrics_path,
|
||||
json.dumps(list(metric_rows), indent=2, sort_keys=True, allow_nan=False)
|
||||
+ "\n",
|
||||
)
|
||||
baselines._atomic_csv(calibration_path, calibration_rows)
|
||||
|
||||
partition_counts, source_counts, label_source_counts = (
|
||||
baselines.manifest_audit_counts(mart, evaluate_locked=evaluate_locked)
|
||||
)
|
||||
manifest = {
|
||||
"model_version": MODEL_VERSION,
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"classification": "private_model_artifact_no_row_predictions",
|
||||
"input_file": mart_path.name,
|
||||
"input_sha256": _input_sha256(mart_path),
|
||||
"metrics_sha256": _input_sha256(metrics_path),
|
||||
"input_rows": mart.input_rows,
|
||||
"eligible_rows": mart.eligible_rows,
|
||||
"partition_counts": partition_counts,
|
||||
"source_era_audit_counts": source_counts,
|
||||
"target_label_source_counts": label_source_counts,
|
||||
"target_contract": baselines.TARGET_CONTRACT,
|
||||
"never_fit_audit_rule": "mart is_vin_audit; validated as vehicle_bucket < 10",
|
||||
"locked_test_evaluated": evaluate_locked,
|
||||
"split_bounds": {
|
||||
name: {
|
||||
"start_inclusive": start.isoformat(),
|
||||
"end_exclusive": end.isoformat(),
|
||||
}
|
||||
for name, (start, end) in baselines.SPLIT_BOUNDS.items()
|
||||
if name != "shadow"
|
||||
},
|
||||
"numeric_features": NUMERIC_FEATURES,
|
||||
"categorical_features": CATEGORICAL_FEATURES,
|
||||
"excluded_from_predictors": EXCLUDED_FROM_PREDICTORS,
|
||||
"preprocessing_fit_partition": "train_2016_2022_non_audit_only",
|
||||
"unknown_category_code": UNKNOWN_CATEGORY_CODE,
|
||||
"selected_candidate": trained.selected_candidate.artifact_state(),
|
||||
"convergence": {
|
||||
"tree_max_iter": trained.max_iter,
|
||||
"tree_selected_n_iter": trained.model_n_iter,
|
||||
"platt_max_iter": trained.calibration_max_iter,
|
||||
"platt_n_iter": trained.platt_n_iter,
|
||||
"requires_early_stop_before_max_iter": True,
|
||||
"requires_finite_scores_and_probabilities": True,
|
||||
},
|
||||
"platt_calibration_config": baselines.PLATT_CALIBRATION_CONFIG,
|
||||
"tuning_results": trained.tuning_results,
|
||||
"seed": seed,
|
||||
"python_version": sys.version,
|
||||
"scikit_learn_version": sklearn_version,
|
||||
"artifacts": [path.name for path in expected],
|
||||
}
|
||||
baselines._atomic_text(
|
||||
manifest_path,
|
||||
json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
)
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
os.umask(0o077)
|
||||
try:
|
||||
mart_path = baselines.require_private_path(args.mart, "--mart")
|
||||
output_dir = baselines.require_private_path(args.output_dir, "--output-dir")
|
||||
learning_rates = _parse_float_grid(
|
||||
args.learning_rates, "--learning-rates", allow_zero=False
|
||||
)
|
||||
leaf_nodes = _parse_int_grid(
|
||||
args.max_leaf_nodes, "--max-leaf-nodes", minimum=2
|
||||
)
|
||||
l2_values = _parse_float_grid(args.l2_grid, "--l2-grid", allow_zero=True)
|
||||
if args.calibration_bins < 2:
|
||||
raise baselines.DataValidationError(
|
||||
"--calibration-bins must be at least 2"
|
||||
)
|
||||
mart = baselines.load_mart(mart_path)
|
||||
trained = train_tree_model(
|
||||
mart=mart,
|
||||
candidates=candidate_grid(learning_rates, leaf_nodes, l2_values),
|
||||
min_category_count=args.min_category_count,
|
||||
max_categories=args.max_categories,
|
||||
min_samples_leaf=args.min_samples_leaf,
|
||||
max_iter=args.max_iter,
|
||||
n_iter_no_change=args.n_iter_no_change,
|
||||
calibration_max_iter=args.calibration_max_iter,
|
||||
seed=args.seed,
|
||||
)
|
||||
metric_rows, calibration_rows = evaluate_tree_model(
|
||||
mart,
|
||||
trained,
|
||||
evaluate_locked=args.evaluate_locked,
|
||||
bin_count=args.calibration_bins,
|
||||
)
|
||||
_np, sklearn, _joblib, _hist, _logistic, _warning = (
|
||||
require_tree_dependencies()
|
||||
)
|
||||
write_artifacts(
|
||||
output_dir=output_dir,
|
||||
mart_path=mart_path,
|
||||
mart=mart,
|
||||
trained=trained,
|
||||
metric_rows=metric_rows,
|
||||
calibration_rows=calibration_rows,
|
||||
evaluate_locked=args.evaluate_locked,
|
||||
seed=args.seed,
|
||||
overwrite=args.overwrite,
|
||||
sklearn_version=sklearn.__version__,
|
||||
)
|
||||
except baselines.BaselineError as exc:
|
||||
print("Tree training failed: {}".format(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
print("Wrote private nonlinear model artifacts to {}".format(output_dir))
|
||||
if args.evaluate_locked:
|
||||
print("The explicitly unlocked 2025 test metrics were evaluated.")
|
||||
else:
|
||||
print("The 2025 locked test was not evaluated.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
39
sql/00_read_only_connection_check.sql
Normal file
39
sql/00_read_only_connection_check.sql
Normal file
@ -0,0 +1,39 @@
|
||||
-- Run the whole file when connected to the countydata database.
|
||||
-- The explicit read-only transaction protects against accidental writes.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
SELECT
|
||||
current_database() AS database_name,
|
||||
current_user AS database_user,
|
||||
current_setting('transaction_read_only') AS transaction_read_only,
|
||||
version() AS postgres_version;
|
||||
|
||||
SELECT
|
||||
ssl,
|
||||
version AS tls_version,
|
||||
cipher,
|
||||
bits
|
||||
FROM pg_stat_ssl
|
||||
WHERE pid = pg_backend_pid();
|
||||
|
||||
SELECT
|
||||
n.nspname AS schema_name,
|
||||
c.relname AS relation_name,
|
||||
CASE c.relkind
|
||||
WHEN 'r' THEN 'table'
|
||||
WHEN 'p' THEN 'partitioned table'
|
||||
WHEN 'v' THEN 'view'
|
||||
WHEN 'm' THEN 'materialized view'
|
||||
WHEN 'f' THEN 'foreign table'
|
||||
ELSE c.relkind::text
|
||||
END AS relation_type,
|
||||
COALESCE(s.n_live_tup, c.reltuples)::bigint AS estimated_rows
|
||||
FROM pg_class AS c
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid
|
||||
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_toast%'
|
||||
ORDER BY n.nspname, c.relname;
|
||||
|
||||
ROLLBACK;
|
||||
53
sql/01_safe_data_overview.sql
Normal file
53
sql/01_safe_data_overview.sql
Normal file
@ -0,0 +1,53 @@
|
||||
-- Aggregate-only starter queries. Do not select VIN, plate, email, session,
|
||||
-- upload-log, or raw JSON fields into a public-facing application.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
SELECT
|
||||
count(*) AS registration_records,
|
||||
min(registration_date) AS earliest_registration,
|
||||
max(registration_date) AS latest_registration
|
||||
FROM data.dmv_tax;
|
||||
|
||||
SELECT
|
||||
extract(year FROM registration_date)::integer AS registration_year,
|
||||
count(*) AS records
|
||||
FROM data.dmv_tax
|
||||
GROUP BY 1
|
||||
ORDER BY 1;
|
||||
|
||||
SELECT
|
||||
upper(trim(county)) AS county,
|
||||
count(*) AS records
|
||||
FROM data.dmv_tax
|
||||
GROUP BY 1
|
||||
ORDER BY 2 DESC;
|
||||
|
||||
SELECT
|
||||
upper(replace(trim(fuel_type), '-', ' ')) AS canonical_fuel_type,
|
||||
count(*) AS records
|
||||
FROM data.dmv_tax_vehicle
|
||||
GROUP BY 1
|
||||
ORDER BY 2 DESC;
|
||||
|
||||
SELECT
|
||||
count(*) AS inspection_records,
|
||||
min(test_start) AS earliest_test,
|
||||
max(test_start) AS latest_test
|
||||
FROM data.inspection_search;
|
||||
|
||||
SELECT
|
||||
extract(year FROM test_start)::integer AS test_year,
|
||||
count(*) AS records
|
||||
FROM data.inspection_search
|
||||
GROUP BY 1
|
||||
ORDER BY 1;
|
||||
|
||||
SELECT
|
||||
lower(trim(county)) AS county_source,
|
||||
coalesce(nullif(upper(trim(overall_result)), ''), '<BLANK/NULL>') AS result,
|
||||
count(*) AS records
|
||||
FROM data.inspection_search
|
||||
GROUP BY 1, 2
|
||||
ORDER BY 1, 3 DESC;
|
||||
|
||||
ROLLBACK;
|
||||
236
sql/10_episode_cohort_feasibility.sql
Normal file
236
sql/10_episode_cohort_feasibility.sql
Normal file
@ -0,0 +1,236 @@
|
||||
-- Aggregate-only feasibility profile for the selected project (label contract v3).
|
||||
--
|
||||
-- This intentionally samples candidate VINs, keeps their complete histories,
|
||||
-- and returns only aggregates. The sample is suitable for cohort development,
|
||||
-- not for final population estimates. Run the whole file on countydata.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
WITH candidate_vins AS MATERIALIZED (
|
||||
SELECT DISTINCT vin
|
||||
FROM data.inspection_search TABLESAMPLE SYSTEM (0.25) REPEATABLE (20260715)
|
||||
WHERE vin IS NOT NULL
|
||||
AND length(btrim(vin)) = 17
|
||||
),
|
||||
sampled_vins AS MATERIALIZED (
|
||||
SELECT vin
|
||||
FROM candidate_vins
|
||||
ORDER BY md5(vin)
|
||||
LIMIT 2000
|
||||
),
|
||||
base AS MATERIALIZED (
|
||||
SELECT
|
||||
s.vin,
|
||||
s.id,
|
||||
s.test_start,
|
||||
lower(btrim(s.county)) AS source_era,
|
||||
CASE
|
||||
WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake'
|
||||
ELSE lower(btrim(s.county))
|
||||
END AS public_county,
|
||||
CASE upper(btrim(s.overall_result))
|
||||
WHEN 'PASS' THEN 'pass'
|
||||
WHEN 'P' THEN 'pass'
|
||||
WHEN 'FAIL' THEN 'fail'
|
||||
WHEN 'F' THEN 'fail'
|
||||
WHEN 'REJECT' THEN 'reject'
|
||||
WHEN 'ABORT' THEN 'abort'
|
||||
ELSE CASE
|
||||
WHEN lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD' THEN
|
||||
CASE upper(btrim(s.obd_result))
|
||||
WHEN 'PASS' THEN 'pass'
|
||||
WHEN 'P' THEN 'pass'
|
||||
WHEN 'FAIL' THEN 'fail'
|
||||
WHEN 'F' THEN 'fail'
|
||||
WHEN 'REJECT' THEN 'reject'
|
||||
WHEN 'ABORT' THEN 'abort'
|
||||
ELSE NULL
|
||||
END
|
||||
ELSE NULL
|
||||
END
|
||||
END AS outcome,
|
||||
CASE
|
||||
WHEN upper(btrim(s.overall_result)) IN (
|
||||
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
||||
) THEN 'overall_result'
|
||||
WHEN lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD'
|
||||
AND upper(btrim(s.obd_result)) IN (
|
||||
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
||||
) THEN 'utah_obd_proxy'
|
||||
ELSE NULL
|
||||
END AS outcome_label_source
|
||||
-- Only Utah OBD/OBD rows may use utah_obd_proxy, and only for binary
|
||||
-- pass/non-pass analysis. Multiclass analysis must require
|
||||
-- outcome_label_source='overall_result'.
|
||||
FROM data.inspection_search AS s
|
||||
JOIN sampled_vins USING (vin)
|
||||
WHERE s.test_start >= timestamp '2010-01-01'
|
||||
),
|
||||
timestamp_quality AS (
|
||||
SELECT
|
||||
vin,
|
||||
test_start,
|
||||
count(
|
||||
DISTINCT row(
|
||||
source_era,
|
||||
public_county,
|
||||
coalesce(outcome, '<unlabeled>'),
|
||||
coalesce(outcome_label_source, '<unlabeled>')
|
||||
)
|
||||
) AS analytical_variants_at_timestamp
|
||||
FROM base
|
||||
GROUP BY 1, 2
|
||||
),
|
||||
timestamp_groups AS (
|
||||
SELECT
|
||||
base.*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vin, test_start
|
||||
ORDER BY id
|
||||
) AS duplicate_rank
|
||||
FROM base
|
||||
),
|
||||
deduplicated AS (
|
||||
SELECT
|
||||
g.vin,
|
||||
g.id,
|
||||
g.test_start,
|
||||
g.source_era,
|
||||
g.public_county,
|
||||
g.outcome,
|
||||
g.outcome_label_source
|
||||
FROM timestamp_groups AS g
|
||||
JOIN timestamp_quality AS q USING (vin, test_start)
|
||||
WHERE g.duplicate_rank = 1
|
||||
AND q.analytical_variants_at_timestamp = 1
|
||||
),
|
||||
with_gaps AS (
|
||||
SELECT
|
||||
*,
|
||||
lag(test_start) OVER (
|
||||
PARTITION BY vin ORDER BY test_start, id
|
||||
) AS previous_test_start
|
||||
FROM deduplicated
|
||||
),
|
||||
episode_markers AS (
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN previous_test_start IS NULL
|
||||
OR test_start - previous_test_start > interval '30 days'
|
||||
THEN 1 ELSE 0
|
||||
END AS starts_episode
|
||||
FROM with_gaps
|
||||
),
|
||||
numbered AS (
|
||||
SELECT
|
||||
*,
|
||||
sum(starts_episode) OVER (
|
||||
PARTITION BY vin ORDER BY test_start, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS episode_number
|
||||
FROM episode_markers
|
||||
),
|
||||
attempts AS (
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vin, episode_number ORDER BY test_start, id
|
||||
) AS attempt_number
|
||||
FROM numbered
|
||||
),
|
||||
episode_summaries AS (
|
||||
SELECT
|
||||
vin,
|
||||
episode_number,
|
||||
min(test_start) AS episode_start,
|
||||
count(*) AS attempt_count,
|
||||
bool_or(outcome = 'pass') AS eventually_passed
|
||||
FROM attempts
|
||||
GROUP BY 1, 2
|
||||
),
|
||||
episodes AS (
|
||||
SELECT
|
||||
a.vin,
|
||||
a.episode_number,
|
||||
a.test_start AS episode_start,
|
||||
a.source_era,
|
||||
a.public_county,
|
||||
a.outcome AS first_outcome,
|
||||
a.outcome_label_source AS first_outcome_label_source,
|
||||
s.attempt_count,
|
||||
s.eventually_passed
|
||||
FROM attempts AS a
|
||||
JOIN episode_summaries AS s USING (vin, episode_number)
|
||||
WHERE a.attempt_number = 1
|
||||
),
|
||||
sequenced AS (
|
||||
SELECT
|
||||
vin,
|
||||
episode_number,
|
||||
episode_start,
|
||||
source_era,
|
||||
public_county,
|
||||
first_outcome,
|
||||
first_outcome_label_source,
|
||||
lag(episode_start) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_episode_start,
|
||||
lag(first_outcome) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_first_outcome,
|
||||
lag(attempt_count) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_attempt_count,
|
||||
sum(attempt_count) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
) AS prior_event_count
|
||||
FROM episodes
|
||||
),
|
||||
eligible AS (
|
||||
SELECT *
|
||||
FROM sequenced
|
||||
WHERE episode_start >= timestamp '2016-01-01'
|
||||
AND episode_number > 1
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_event_count <= 50
|
||||
)
|
||||
SELECT
|
||||
CASE
|
||||
WHEN grouping(extract(year FROM episode_start)::integer) = 1
|
||||
THEN NULL
|
||||
ELSE extract(year FROM episode_start)::integer
|
||||
END AS target_year,
|
||||
count(*) AS eligible_episodes,
|
||||
count(DISTINCT vin) AS vehicles,
|
||||
count(*) FILTER (WHERE first_outcome = 'pass') AS pass_episodes,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) AS nonpass_episodes,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome_label_source = 'utah_obd_proxy'
|
||||
) AS utah_obd_proxy_episodes,
|
||||
round(
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
)::numeric / nullif(count(*), 0),
|
||||
4
|
||||
) AS nonpass_rate,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY extract(epoch FROM (episode_start-prior_episode_start))/86400.0
|
||||
) AS median_days_since_prior_episode,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY prior_attempt_count
|
||||
) AS median_prior_attempts
|
||||
FROM eligible
|
||||
GROUP BY GROUPING SETS (
|
||||
(),
|
||||
(extract(year FROM episode_start)::integer)
|
||||
)
|
||||
ORDER BY target_year NULLS FIRST;
|
||||
|
||||
ROLLBACK;
|
||||
114
sql/11_outcome_mapping_audit.sql
Normal file
114
sql/11_outcome_mapping_audit.sql
Normal file
@ -0,0 +1,114 @@
|
||||
-- Aggregate-only audit of source-specific inspection result encodings.
|
||||
--
|
||||
-- Values are controlled inspection categories, never identifiers. Counts below
|
||||
-- 100 are suppressed so accidental free-text anomalies cannot be surfaced.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
WITH normalized AS MATERIALIZED (
|
||||
SELECT
|
||||
lower(coalesce(nullif(btrim(county), ''), '<blank/null>')) AS source_era,
|
||||
upper(coalesce(nullif(btrim(overall_result), ''), '<BLANK/NULL>'))
|
||||
AS overall_result,
|
||||
upper(coalesce(nullif(btrim(obd_result), ''), '<BLANK/NULL>'))
|
||||
AS obd_result,
|
||||
lower(coalesce(nullif(btrim(program_type), ''), '<blank/null>'))
|
||||
AS program_type,
|
||||
upper(coalesce(nullif(btrim(test_type), ''), '<BLANK/NULL>'))
|
||||
AS test_type
|
||||
FROM data.inspection_search
|
||||
WHERE test_start >= timestamp '2010-01-01'
|
||||
),
|
||||
overall_counts AS (
|
||||
SELECT
|
||||
'source_overall_result'::text AS audit_section,
|
||||
source_era,
|
||||
overall_result AS value_1,
|
||||
CASE
|
||||
WHEN overall_result IN ('PASS', 'P') THEN 'pass'
|
||||
WHEN overall_result IN ('FAIL', 'F') THEN 'fail'
|
||||
WHEN overall_result = 'REJECT' THEN 'reject'
|
||||
WHEN overall_result = 'ABORT' THEN 'abort'
|
||||
ELSE '<unrecognized>'
|
||||
END::text AS value_2,
|
||||
NULL::text AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
GROUP BY 1, 2, 3, 4
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
utah_unrecognized_context AS (
|
||||
SELECT
|
||||
'utah_unrecognized_context'::text AS audit_section,
|
||||
source_era,
|
||||
overall_result AS value_1,
|
||||
obd_result AS value_2,
|
||||
program_type || ' / ' || test_type AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
WHERE source_era = 'utah'
|
||||
AND overall_result NOT IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
overall_obd_crosscheck AS (
|
||||
SELECT
|
||||
'recognized_overall_vs_obd'::text AS audit_section,
|
||||
source_era,
|
||||
overall_result AS value_1,
|
||||
obd_result AS value_2,
|
||||
program_type || ' / ' || test_type AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
WHERE overall_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
AND obd_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
binary_crosscheck AS (
|
||||
SELECT
|
||||
'binary_overall_vs_obd'::text AS audit_section,
|
||||
source_era,
|
||||
CASE
|
||||
WHEN (overall_result IN ('PASS', 'P'))
|
||||
= (obd_result IN ('PASS', 'P'))
|
||||
THEN 'agree'
|
||||
ELSE 'disagree'
|
||||
END::text AS value_1,
|
||||
NULL::text AS value_2,
|
||||
NULL::text AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
WHERE overall_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
AND obd_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
GROUP BY 1, 2, 3
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
ranked AS (
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY audit_section, source_era
|
||||
ORDER BY records DESC, value_1, value_2, value_3
|
||||
) AS frequency_rank
|
||||
FROM (
|
||||
SELECT * FROM overall_counts
|
||||
UNION ALL
|
||||
SELECT * FROM utah_unrecognized_context
|
||||
UNION ALL
|
||||
SELECT * FROM overall_obd_crosscheck
|
||||
UNION ALL
|
||||
SELECT * FROM binary_crosscheck
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
audit_section,
|
||||
source_era,
|
||||
value_1,
|
||||
value_2,
|
||||
value_3,
|
||||
records
|
||||
FROM ranked
|
||||
WHERE frequency_rank <= 50
|
||||
ORDER BY audit_section, source_era, records DESC;
|
||||
|
||||
ROLLBACK;
|
||||
152
sql/local/20_stage_events.sql
Normal file
152
sql/local/20_stage_events.sql
Normal file
@ -0,0 +1,152 @@
|
||||
-- Normalize explicitly all-VARCHAR CSV staging rows, remove exact duplicate
|
||||
-- analytical records, and quarantine timestamps whose event order is ambiguous.
|
||||
|
||||
CREATE OR REPLACE TABLE normalized_events AS
|
||||
SELECT
|
||||
batch_file,
|
||||
batch_kind,
|
||||
batch_start,
|
||||
batch_end,
|
||||
try_cast(internal_event_id AS BIGINT) AS internal_event_id,
|
||||
lower(vehicle_token) AS vehicle_token,
|
||||
try_cast(vehicle_bucket AS INTEGER) AS vehicle_bucket,
|
||||
try_cast(event_ts AS TIMESTAMP) AS event_ts,
|
||||
lower(nullif(trim(source_era), '')) AS source_era,
|
||||
lower(nullif(trim(public_county), '')) AS public_county,
|
||||
lower(nullif(trim(canonical_outcome), '')) AS canonical_outcome,
|
||||
lower(nullif(trim(outcome_label_source), '')) AS outcome_label_source,
|
||||
lower(nullif(trim(program_type), '')) AS program_type,
|
||||
upper(nullif(trim(test_type), '')) AS test_type,
|
||||
upper(nullif(trim(observed_make), '')) AS observed_make,
|
||||
upper(nullif(trim(observed_model), '')) AS observed_model,
|
||||
try_cast(observed_model_year AS INTEGER) AS observed_model_year,
|
||||
CASE
|
||||
WHEN try_cast(internal_event_id AS BIGINT) IS NULL
|
||||
THEN 'invalid_internal_event_id'
|
||||
WHEN vehicle_token IS NULL
|
||||
OR NOT regexp_full_match(lower(vehicle_token), '^[0-9a-f]{64}$')
|
||||
THEN 'invalid_vehicle_token'
|
||||
WHEN try_cast(vehicle_bucket AS INTEGER) IS NULL
|
||||
OR try_cast(vehicle_bucket AS INTEGER) NOT BETWEEN 0 AND 99
|
||||
THEN 'invalid_vehicle_bucket'
|
||||
WHEN try_cast(event_ts AS TIMESTAMP) IS NULL
|
||||
THEN 'invalid_event_timestamp'
|
||||
WHEN try_cast(event_ts AS TIMESTAMP) < TIMESTAMP '2010-01-01'
|
||||
THEN 'pre_2010_timestamp'
|
||||
WHEN try_cast(event_ts AS TIMESTAMP) < batch_start
|
||||
OR try_cast(event_ts AS TIMESTAMP) >= batch_end
|
||||
THEN 'timestamp_outside_manifest_range'
|
||||
WHEN lower(nullif(trim(canonical_outcome), '')) IS NOT NULL
|
||||
AND lower(nullif(trim(canonical_outcome), ''))
|
||||
NOT IN ('pass', 'fail', 'reject', 'abort')
|
||||
THEN 'invalid_canonical_outcome'
|
||||
WHEN lower(nullif(trim(outcome_label_source), '')) IS NOT NULL
|
||||
AND lower(nullif(trim(outcome_label_source), ''))
|
||||
NOT IN ('overall_result', 'utah_obd_proxy')
|
||||
THEN 'invalid_outcome_label_source'
|
||||
WHEN lower(nullif(trim(canonical_outcome), ''))
|
||||
IN ('pass', 'fail', 'reject', 'abort')
|
||||
AND lower(nullif(trim(outcome_label_source), '')) IS NULL
|
||||
THEN 'missing_outcome_label_source'
|
||||
WHEN lower(nullif(trim(outcome_label_source), '')) = 'utah_obd_proxy'
|
||||
AND lower(nullif(trim(source_era), '')) IS DISTINCT FROM 'utah'
|
||||
THEN 'utah_proxy_non_utah_source'
|
||||
WHEN lower(nullif(trim(source_era), '')) IN ('slc', 'slco')
|
||||
AND lower(nullif(trim(public_county), ''))
|
||||
IS DISTINCT FROM 'salt_lake'
|
||||
THEN 'inconsistent_public_county'
|
||||
ELSE NULL
|
||||
END AS invalid_reason
|
||||
FROM stg_events;
|
||||
|
||||
CREATE OR REPLACE TABLE ranked_events AS
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY
|
||||
vehicle_token,
|
||||
vehicle_bucket,
|
||||
event_ts,
|
||||
source_era,
|
||||
public_county,
|
||||
canonical_outcome,
|
||||
outcome_label_source,
|
||||
program_type,
|
||||
test_type,
|
||||
observed_make,
|
||||
observed_model,
|
||||
observed_model_year
|
||||
ORDER BY internal_event_id, batch_file
|
||||
) AS exact_duplicate_rank
|
||||
FROM normalized_events;
|
||||
|
||||
CREATE OR REPLACE TABLE conflicting_timestamps AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
event_ts,
|
||||
count(*) AS distinct_event_count
|
||||
FROM ranked_events
|
||||
WHERE invalid_reason IS NULL
|
||||
AND exact_duplicate_rank = 1
|
||||
GROUP BY vehicle_token, event_ts
|
||||
HAVING count(*) > 1;
|
||||
|
||||
CREATE OR REPLACE TABLE event_exclusions AS
|
||||
SELECT
|
||||
internal_event_id,
|
||||
vehicle_token,
|
||||
event_ts,
|
||||
batch_file,
|
||||
invalid_reason AS exclusion_reason
|
||||
FROM ranked_events
|
||||
WHERE invalid_reason IS NOT NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
internal_event_id,
|
||||
vehicle_token,
|
||||
event_ts,
|
||||
batch_file,
|
||||
'exact_duplicate' AS exclusion_reason
|
||||
FROM ranked_events
|
||||
WHERE invalid_reason IS NULL
|
||||
AND exact_duplicate_rank > 1
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
r.internal_event_id,
|
||||
r.vehicle_token,
|
||||
r.event_ts,
|
||||
r.batch_file,
|
||||
'conflicting_same_timestamp' AS exclusion_reason
|
||||
FROM ranked_events AS r
|
||||
JOIN conflicting_timestamps AS c
|
||||
ON c.vehicle_token = r.vehicle_token
|
||||
AND c.event_ts = r.event_ts
|
||||
WHERE r.invalid_reason IS NULL
|
||||
AND r.exact_duplicate_rank = 1;
|
||||
|
||||
CREATE OR REPLACE TABLE clean_events AS
|
||||
SELECT
|
||||
r.internal_event_id,
|
||||
r.vehicle_token,
|
||||
r.vehicle_bucket,
|
||||
r.event_ts,
|
||||
r.source_era,
|
||||
r.public_county,
|
||||
r.canonical_outcome,
|
||||
r.outcome_label_source,
|
||||
r.program_type,
|
||||
r.test_type,
|
||||
r.observed_make,
|
||||
r.observed_model,
|
||||
r.observed_model_year
|
||||
FROM ranked_events AS r
|
||||
LEFT JOIN conflicting_timestamps AS c
|
||||
ON c.vehicle_token = r.vehicle_token
|
||||
AND c.event_ts = r.event_ts
|
||||
WHERE r.invalid_reason IS NULL
|
||||
AND r.exact_duplicate_rank = 1
|
||||
AND c.vehicle_token IS NULL;
|
||||
115
sql/local/21_build_episodes.sql
Normal file
115
sql/local/21_build_episodes.sql
Normal file
@ -0,0 +1,115 @@
|
||||
-- Construct episodes across the complete, globally ordered input history.
|
||||
-- A gap exactly equal to the configured threshold remains in the same episode.
|
||||
|
||||
CREATE OR REPLACE TABLE sequenced_events AS
|
||||
WITH with_previous AS (
|
||||
SELECT
|
||||
*,
|
||||
lag(event_ts) OVER (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY event_ts, internal_event_id
|
||||
) AS previous_event_ts
|
||||
FROM clean_events
|
||||
),
|
||||
with_markers AS (
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN previous_event_ts IS NULL THEN 1
|
||||
WHEN event_ts - previous_event_ts
|
||||
> (SELECT episode_gap_days FROM build_config)
|
||||
* INTERVAL '1 day'
|
||||
THEN 1
|
||||
ELSE 0
|
||||
END AS starts_episode
|
||||
FROM with_previous
|
||||
),
|
||||
with_episode_number AS (
|
||||
SELECT
|
||||
*,
|
||||
sum(starts_episode) OVER (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY event_ts, internal_event_id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS episode_number
|
||||
FROM with_markers
|
||||
)
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vehicle_token, episode_number
|
||||
ORDER BY event_ts, internal_event_id
|
||||
) AS attempt_number,
|
||||
row_number() OVER (
|
||||
PARTITION BY vehicle_token, episode_number
|
||||
ORDER BY event_ts DESC, internal_event_id DESC
|
||||
) AS reverse_attempt_number
|
||||
FROM with_episode_number;
|
||||
|
||||
CREATE OR REPLACE TABLE episode_daily_activity AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
episode_number,
|
||||
cast(event_ts AS DATE) AS event_date,
|
||||
count(*) AS events_in_day
|
||||
FROM sequenced_events
|
||||
GROUP BY vehicle_token, episode_number, cast(event_ts AS DATE);
|
||||
|
||||
CREATE OR REPLACE TABLE episode_aggregates AS
|
||||
SELECT
|
||||
e.vehicle_token,
|
||||
e.vehicle_bucket,
|
||||
e.episode_number,
|
||||
min(e.event_ts) AS episode_start,
|
||||
max(e.event_ts) AS episode_end,
|
||||
count(*) AS attempt_count,
|
||||
count(e.canonical_outcome) AS labeled_attempt_count,
|
||||
coalesce(bool_or(e.canonical_outcome = 'pass'), false) AS eventually_passed,
|
||||
max(d.events_in_day) AS max_events_in_day,
|
||||
(list(e.observed_make ORDER BY e.event_ts DESC, e.internal_event_id DESC)
|
||||
FILTER (WHERE e.observed_make IS NOT NULL))[1]
|
||||
AS episode_last_observed_make,
|
||||
(list(e.observed_model ORDER BY e.event_ts DESC, e.internal_event_id DESC)
|
||||
FILTER (WHERE e.observed_model IS NOT NULL))[1]
|
||||
AS episode_last_observed_model,
|
||||
(list(e.observed_model_year ORDER BY e.event_ts DESC, e.internal_event_id DESC)
|
||||
FILTER (WHERE e.observed_model_year IS NOT NULL))[1]
|
||||
AS episode_last_observed_model_year
|
||||
FROM sequenced_events AS e
|
||||
JOIN episode_daily_activity AS d
|
||||
ON d.vehicle_token = e.vehicle_token
|
||||
AND d.episode_number = e.episode_number
|
||||
AND d.event_date = cast(e.event_ts AS DATE)
|
||||
GROUP BY e.vehicle_token, e.vehicle_bucket, e.episode_number;
|
||||
|
||||
CREATE OR REPLACE TABLE inspection_episodes AS
|
||||
SELECT
|
||||
a.vehicle_token,
|
||||
a.vehicle_bucket,
|
||||
a.episode_number,
|
||||
a.episode_start,
|
||||
a.episode_end,
|
||||
f.internal_event_id AS first_internal_event_id,
|
||||
f.source_era,
|
||||
f.public_county,
|
||||
f.canonical_outcome AS first_outcome,
|
||||
f.outcome_label_source AS first_outcome_label_source,
|
||||
f.program_type AS first_program_type,
|
||||
f.test_type AS first_test_type,
|
||||
l.canonical_outcome AS final_outcome,
|
||||
a.attempt_count,
|
||||
a.labeled_attempt_count,
|
||||
a.eventually_passed,
|
||||
a.max_events_in_day,
|
||||
a.episode_last_observed_make,
|
||||
a.episode_last_observed_model,
|
||||
a.episode_last_observed_model_year
|
||||
FROM episode_aggregates AS a
|
||||
JOIN sequenced_events AS f
|
||||
ON f.vehicle_token = a.vehicle_token
|
||||
AND f.episode_number = a.episode_number
|
||||
AND f.attempt_number = 1
|
||||
JOIN sequenced_events AS l
|
||||
ON l.vehicle_token = a.vehicle_token
|
||||
AND l.episode_number = a.episode_number
|
||||
AND l.reverse_attempt_number = 1;
|
||||
209
sql/local/22_build_features.sql
Normal file
209
sql/local/22_build_features.sql
Normal file
@ -0,0 +1,209 @@
|
||||
-- Build one explicitly point-in-time row per episode. All predictors carrying
|
||||
-- history use a window ending at one episode preceding the target.
|
||||
|
||||
CREATE OR REPLACE TABLE episode_history AS
|
||||
WITH history_windows AS (
|
||||
SELECT
|
||||
e.*,
|
||||
count(*) OVER prior_all AS prior_episode_count,
|
||||
coalesce(sum(attempt_count) OVER prior_all, 0) AS prior_total_attempt_count,
|
||||
count(first_outcome) OVER prior_all AS prior_labeled_episode_count,
|
||||
lag(attempt_count) OVER by_vehicle AS prior_attempt_count,
|
||||
lag(first_outcome) OVER by_vehicle AS prior_first_outcome,
|
||||
lag(final_outcome) OVER by_vehicle AS prior_final_outcome,
|
||||
lag(episode_start) OVER by_vehicle AS prior_episode_start,
|
||||
max(CASE
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort')
|
||||
THEN episode_start
|
||||
END) OVER prior_all AS prior_adverse_episode_start,
|
||||
count(*) FILTER (WHERE first_outcome = 'pass') OVER prior_all
|
||||
AS prior_pass_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'fail') OVER prior_all
|
||||
AS prior_fail_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'reject') OVER prior_all
|
||||
AS prior_reject_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'abort') OVER prior_all
|
||||
AS prior_abort_count,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) OVER prior_all AS prior_nonpass_count,
|
||||
count(first_outcome) OVER prior_three AS last3_labeled_episode_count,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) OVER prior_three AS last3_nonpass_count,
|
||||
max(max_events_in_day) OVER prior_all AS prior_max_events_in_day,
|
||||
arg_max(episode_last_observed_make, episode_number)
|
||||
FILTER (WHERE episode_last_observed_make IS NOT NULL)
|
||||
OVER prior_all AS last_observed_make,
|
||||
arg_max(episode_last_observed_model, episode_number)
|
||||
FILTER (WHERE episode_last_observed_model IS NOT NULL)
|
||||
OVER prior_all AS last_observed_model,
|
||||
arg_max(episode_last_observed_model_year, episode_number)
|
||||
FILTER (WHERE episode_last_observed_model_year IS NOT NULL)
|
||||
OVER prior_all AS last_observed_model_year
|
||||
FROM inspection_episodes AS e
|
||||
WINDOW
|
||||
by_vehicle AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
),
|
||||
prior_all AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
),
|
||||
prior_three AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
ROWS BETWEEN 3 PRECEDING AND 1 PRECEDING
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN prior_episode_start IS NULL THEN NULL
|
||||
ELSE extract(epoch FROM (episode_start - prior_episode_start)) / 86400.0
|
||||
END AS days_since_prior_episode,
|
||||
CASE
|
||||
WHEN prior_adverse_episode_start IS NULL THEN NULL
|
||||
ELSE extract(epoch FROM (episode_start - prior_adverse_episode_start))
|
||||
/ 86400.0
|
||||
END AS days_since_prior_adverse,
|
||||
prior_nonpass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
||||
AS prior_nonpass_rate,
|
||||
prior_pass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
||||
AS prior_pass_rate,
|
||||
last3_nonpass_count::DOUBLE / nullif(last3_labeled_episode_count, 0)
|
||||
AS last3_nonpass_rate
|
||||
FROM history_windows;
|
||||
|
||||
CREATE OR REPLACE TABLE feature_mart AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
vehicle_bucket,
|
||||
vehicle_bucket < 10 AS is_vin_audit,
|
||||
episode_number::BIGINT AS episode_number,
|
||||
episode_start,
|
||||
first_outcome,
|
||||
first_outcome_label_source AS target_outcome_label_source,
|
||||
CASE
|
||||
WHEN first_outcome = 'pass' THEN 0
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
|
||||
ELSE NULL
|
||||
END::INTEGER AS target_nonpass,
|
||||
episode_start >= TIMESTAMP '2016-01-01'
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_episode_count >= 1
|
||||
AND prior_total_attempt_count <= 50
|
||||
AND coalesce(prior_max_events_in_day, 0) <= 4
|
||||
AS eligible_returning_target,
|
||||
CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
|
||||
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
|
||||
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
|
||||
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
|
||||
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
|
||||
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
|
||||
ELSE 'out_of_scope'
|
||||
END AS temporal_partition,
|
||||
CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'before_target_period'
|
||||
WHEN first_outcome IS NULL THEN 'unlabeled_first_outcome'
|
||||
WHEN prior_episode_count < 1 THEN 'cold_start'
|
||||
WHEN prior_total_attempt_count > 50 THEN 'prior_event_count_over_50'
|
||||
WHEN coalesce(prior_max_events_in_day, 0) > 4
|
||||
THEN 'prior_daily_activity_over_4'
|
||||
ELSE NULL
|
||||
END AS eligibility_exclusion_reason,
|
||||
public_county,
|
||||
source_era,
|
||||
CASE
|
||||
WHEN month(episode_start) IN (12, 1, 2) THEN 'winter'
|
||||
WHEN month(episode_start) IN (3, 4, 5) THEN 'spring'
|
||||
WHEN month(episode_start) IN (6, 7, 8) THEN 'summer'
|
||||
ELSE 'fall'
|
||||
END AS target_season,
|
||||
CASE
|
||||
WHEN last_observed_model_year BETWEEN 1886 AND year(episode_start) + 1
|
||||
THEN greatest(year(episode_start) - last_observed_model_year, 0)
|
||||
ELSE NULL
|
||||
END::INTEGER AS vehicle_age,
|
||||
prior_episode_count,
|
||||
prior_total_attempt_count::BIGINT AS prior_total_attempt_count,
|
||||
prior_attempt_count,
|
||||
days_since_prior_episode,
|
||||
days_since_prior_adverse,
|
||||
prior_nonpass_rate,
|
||||
prior_first_outcome,
|
||||
prior_final_outcome,
|
||||
last_observed_make,
|
||||
last_observed_model,
|
||||
last_observed_model_year,
|
||||
prior_labeled_episode_count,
|
||||
prior_pass_count,
|
||||
prior_fail_count,
|
||||
prior_reject_count,
|
||||
prior_abort_count,
|
||||
prior_nonpass_count,
|
||||
prior_pass_rate,
|
||||
last3_labeled_episode_count,
|
||||
last3_nonpass_count,
|
||||
last3_nonpass_rate,
|
||||
prior_max_events_in_day,
|
||||
prior_first_outcome IS NULL AS prior_first_outcome_missing,
|
||||
last_observed_model_year IS NULL AS prior_model_year_missing,
|
||||
(SELECT source_data_kind FROM build_config) AS source_data_kind,
|
||||
(SELECT population_estimate_allowed FROM build_config)
|
||||
AS population_estimate_allowed
|
||||
FROM episode_history;
|
||||
|
||||
CREATE OR REPLACE TABLE cohort_flow AS
|
||||
SELECT
|
||||
'event' AS grain,
|
||||
'input' AS stage,
|
||||
'all_rows' AS reason,
|
||||
count(*) AS observation_count,
|
||||
count(DISTINCT try_cast(vehicle_token AS VARCHAR)) AS vehicle_count
|
||||
FROM stg_events
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'event',
|
||||
'excluded',
|
||||
exclusion_reason,
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM event_exclusions
|
||||
GROUP BY exclusion_reason
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'event',
|
||||
'accepted',
|
||||
'clean_events',
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM clean_events
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode',
|
||||
'constructed',
|
||||
'all_episodes',
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM inspection_episodes
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode',
|
||||
'eligible_returning_target',
|
||||
coalesce(eligibility_exclusion_reason, 'eligible'),
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM feature_mart
|
||||
GROUP BY coalesce(eligibility_exclusion_reason, 'eligible');
|
||||
155
sql/local/23_validate_mart.sql
Normal file
155
sql/local/23_validate_mart.sql
Normal file
@ -0,0 +1,155 @@
|
||||
-- Every row reports a count of invariant violations. The Python driver refuses
|
||||
-- to publish the mart unless every count is zero.
|
||||
|
||||
CREATE OR REPLACE TABLE mart_validation AS
|
||||
SELECT
|
||||
'clean_internal_event_id_unique' AS check_name,
|
||||
count(*) AS violation_count
|
||||
FROM (
|
||||
SELECT internal_event_id
|
||||
FROM clean_events
|
||||
GROUP BY internal_event_id
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode_key_unique',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT vehicle_token, episode_number
|
||||
FROM inspection_episodes
|
||||
GROUP BY vehicle_token, episode_number
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'mart_key_unique',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT vehicle_token, episode_number
|
||||
FROM feature_mart
|
||||
GROUP BY vehicle_token, episode_number
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode_attempts_reconcile',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT
|
||||
e.vehicle_token,
|
||||
e.episode_number
|
||||
FROM inspection_episodes AS e
|
||||
JOIN (
|
||||
SELECT vehicle_token, episode_number, count(*) AS actual_attempts
|
||||
FROM sequenced_events
|
||||
GROUP BY vehicle_token, episode_number
|
||||
) AS a USING (vehicle_token, episode_number)
|
||||
WHERE e.attempt_count <> a.actual_attempts
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode_gap_strictly_greater_than_threshold',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT
|
||||
episode_start,
|
||||
lag(episode_end) OVER (
|
||||
PARTITION BY vehicle_token ORDER BY episode_number
|
||||
) AS prior_episode_end
|
||||
FROM inspection_episodes
|
||||
) AS gaps
|
||||
WHERE prior_episode_end IS NOT NULL
|
||||
AND episode_start - prior_episode_end
|
||||
<= (SELECT episode_gap_days FROM build_config) * INTERVAL '1 day'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'prior_episode_count_point_in_time',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE prior_episode_count <> episode_number - 1
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'target_mapping_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE target_nonpass IS DISTINCT FROM CASE
|
||||
WHEN first_outcome = 'pass' THEN 0
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
|
||||
ELSE NULL
|
||||
END
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'target_outcome_label_source_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE (first_outcome IS NOT NULL AND (
|
||||
target_outcome_label_source IS NULL
|
||||
OR target_outcome_label_source
|
||||
NOT IN ('overall_result', 'utah_obd_proxy')
|
||||
))
|
||||
OR (target_outcome_label_source = 'utah_obd_proxy'
|
||||
AND source_era IS DISTINCT FROM 'utah')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'raw_obd_result_absent_from_mart',
|
||||
count(*)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'main'
|
||||
AND table_name = 'feature_mart'
|
||||
AND lower(column_name) = 'obd_result'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'audit_bucket_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE vehicle_bucket NOT BETWEEN 0 AND 99
|
||||
OR is_vin_audit IS DISTINCT FROM (vehicle_bucket < 10)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'eligibility_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE eligible_returning_target IS DISTINCT FROM (
|
||||
episode_start >= TIMESTAMP '2016-01-01'
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_episode_count >= 1
|
||||
AND prior_total_attempt_count <= 50
|
||||
AND coalesce(prior_max_events_in_day, 0) <= 4
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'temporal_partition_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE temporal_partition IS DISTINCT FROM CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
|
||||
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
|
||||
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
|
||||
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
|
||||
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
|
||||
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
|
||||
ELSE 'out_of_scope'
|
||||
END;
|
||||
545
tests/test_baselines.py
Normal file
545
tests/test_baselines.py
Normal file
@ -0,0 +1,545 @@
|
||||
"""Tests for the private, leakage-safe baseline training contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import gzip
|
||||
import importlib.util
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Mapping
|
||||
from unittest import mock
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE_PATH = PROJECT_ROOT / "scripts/train_baselines.py"
|
||||
SPEC = importlib.util.spec_from_file_location("train_baselines", MODULE_PATH)
|
||||
if SPEC is None or SPEC.loader is None: # pragma: no cover - import machinery guard
|
||||
raise RuntimeError("Could not load scripts/train_baselines.py")
|
||||
baselines = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = baselines
|
||||
SPEC.loader.exec_module(baselines)
|
||||
|
||||
|
||||
SKLEARN_AVAILABLE = importlib.util.find_spec("numpy") is not None and importlib.util.find_spec(
|
||||
"sklearn"
|
||||
) is not None
|
||||
DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None
|
||||
|
||||
|
||||
def valid_record(
|
||||
episode_start: str = "2020-06-01T09:00:00",
|
||||
partition: str = "train",
|
||||
target: int = 0,
|
||||
bucket: int = 25,
|
||||
token_number: int = 1,
|
||||
) -> Dict[str, object]:
|
||||
return {
|
||||
"vehicle_token": "{:064x}".format(token_number),
|
||||
"vehicle_bucket": bucket,
|
||||
"is_vin_audit": "true" if bucket < 10 else "false",
|
||||
"episode_number": 2,
|
||||
"episode_start": episode_start,
|
||||
"first_outcome": "fail" if target else "pass",
|
||||
"target_nonpass": target,
|
||||
"eligible_returning_target": "true",
|
||||
"temporal_partition": partition,
|
||||
"vehicle_age": 10,
|
||||
"prior_episode_count": 1,
|
||||
"prior_total_attempt_count": 1,
|
||||
"prior_attempt_count": 1,
|
||||
"days_since_prior_episode": 365,
|
||||
"days_since_prior_adverse": "",
|
||||
"prior_nonpass_rate": 0,
|
||||
"public_county": "salt_lake",
|
||||
"source_era": "slc",
|
||||
"target_outcome_label_source": "overall_result",
|
||||
"target_season": "summer",
|
||||
"prior_first_outcome": "pass",
|
||||
"prior_final_outcome": "pass",
|
||||
"last_observed_make": "toyota",
|
||||
"last_observed_model": "camry",
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path: Path, records: Iterable[Mapping[str, object]]) -> None:
|
||||
rows = list(records)
|
||||
if not rows:
|
||||
raise ValueError("Tests require at least one record")
|
||||
opener = gzip.open if path.name.endswith(".gz") else open
|
||||
with opener(path, "wt", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
class SchemaTests(unittest.TestCase):
|
||||
def test_required_schema_is_accepted(self) -> None:
|
||||
baselines.validate_schema(list(valid_record().keys()))
|
||||
|
||||
def test_missing_required_column_fails_closed(self) -> None:
|
||||
columns = list(valid_record().keys())
|
||||
columns.remove("prior_first_outcome")
|
||||
with self.assertRaisesRegex(baselines.SchemaError, "prior_first_outcome"):
|
||||
baselines.validate_schema(columns)
|
||||
|
||||
def test_target_label_source_column_is_required(self) -> None:
|
||||
columns = list(valid_record().keys())
|
||||
columns.remove("target_outcome_label_source")
|
||||
with self.assertRaisesRegex(
|
||||
baselines.SchemaError, "target_outcome_label_source"
|
||||
):
|
||||
baselines.validate_schema(columns)
|
||||
|
||||
def test_forbidden_column_fails_even_when_not_a_feature(self) -> None:
|
||||
columns = list(valid_record().keys()) + ["vin"]
|
||||
with self.assertRaisesRegex(baselines.SchemaError, "forbidden"):
|
||||
baselines.validate_schema(columns)
|
||||
|
||||
def test_unknown_sensitive_column_fails_closed(self) -> None:
|
||||
columns = list(valid_record().keys()) + ["owner_zip_code"]
|
||||
with self.assertRaisesRegex(baselines.SchemaError, "owner_zip_code"):
|
||||
baselines.validate_schema(columns)
|
||||
|
||||
def test_duplicate_columns_fail_closed(self) -> None:
|
||||
columns = list(valid_record().keys()) + ["vehicle_token"]
|
||||
with self.assertRaisesRegex(baselines.SchemaError, "duplicate"):
|
||||
baselines.validate_schema(columns)
|
||||
|
||||
|
||||
class MartValidationTests(unittest.TestCase):
|
||||
def test_loads_csv_gzip_and_assigns_fixed_partitions(self) -> None:
|
||||
records = [
|
||||
valid_record("2022-12-31T23:59:59", "train", 0, 25, 1),
|
||||
valid_record("2023-01-01T00:00:00", "tune", 1, 25, 2),
|
||||
valid_record("2024-01-01T00:00:00", "calibrate", 0, 5, 3),
|
||||
valid_record("2025-01-01T00:00:00", "test", 1, 25, 4),
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "mart.csv.gz"
|
||||
write_csv(path, records)
|
||||
mart = baselines.load_mart(path)
|
||||
self.assertEqual(mart.eligible_rows, 4)
|
||||
self.assertEqual(len(mart.rows_by_partition["train"]), 1)
|
||||
self.assertEqual(len(mart.rows_by_partition["tune"]), 1)
|
||||
self.assertEqual(len(mart.rows_by_partition["calibrate"]), 1)
|
||||
self.assertEqual(len(mart.rows_by_partition["locked_test"]), 1)
|
||||
self.assertTrue(mart.rows_by_partition["calibrate"][0].audit_vehicle)
|
||||
|
||||
@unittest.skipUnless(DUCKDB_AVAILABLE, "duckdb is not installed")
|
||||
def test_loads_parquet_through_duckdb_without_pyarrow(self) -> None:
|
||||
import duckdb
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
csv_path = Path(directory) / "mart.csv"
|
||||
parquet_path = Path(directory) / "mart.parquet"
|
||||
write_csv(csv_path, [valid_record()])
|
||||
connection = duckdb.connect(database=":memory:")
|
||||
try:
|
||||
connection.execute(
|
||||
"CREATE TABLE mart AS SELECT * FROM read_csv_auto(?)", [str(csv_path)]
|
||||
)
|
||||
escaped_path = str(parquet_path).replace("'", "''")
|
||||
connection.execute(
|
||||
"COPY mart TO '{}' (FORMAT PARQUET)".format(escaped_path)
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
mart = baselines.load_mart(parquet_path)
|
||||
self.assertEqual(mart.eligible_rows, 1)
|
||||
self.assertEqual(len(mart.rows_by_partition["train"]), 1)
|
||||
|
||||
def test_declared_partition_must_match_timestamp(self) -> None:
|
||||
record = valid_record("2025-05-01T00:00:00", "train")
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "belongs to locked_test"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_label_column_must_match_canonical_outcome(self) -> None:
|
||||
record = valid_record(target=1)
|
||||
record["target_nonpass"] = 0
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "disagrees"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_audit_boolean_must_match_bucket_contract(self) -> None:
|
||||
record = valid_record(bucket=5)
|
||||
record["is_vin_audit"] = "false"
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "bucket<10"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_source_era_is_required_for_audit_but_not_modeling(self) -> None:
|
||||
record = valid_record()
|
||||
record["source_era"] = ""
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "drift auditing"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_utah_proxy_is_accepted_only_for_utah_source_era(self) -> None:
|
||||
record = valid_record()
|
||||
record["source_era"] = "utah"
|
||||
record["target_outcome_label_source"] = "utah_obd_proxy"
|
||||
episode = baselines.episode_from_record(record, 2)
|
||||
self.assertEqual(episode.target_outcome_label_source, "utah_obd_proxy")
|
||||
|
||||
record["source_era"] = "slc"
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "outside source_era"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
record["source_era"] = "utah"
|
||||
record["first_outcome"] = "reject"
|
||||
record["target_nonpass"] = 1
|
||||
rejected = baselines.episode_from_record(record, 2)
|
||||
self.assertEqual(rejected.target, 1)
|
||||
self.assertEqual(rejected.target_outcome_label_source, "utah_obd_proxy")
|
||||
|
||||
record["first_outcome"] = "abort"
|
||||
aborted = baselines.episode_from_record(record, 2)
|
||||
self.assertEqual(aborted.target, 1)
|
||||
self.assertEqual(aborted.target_outcome_label_source, "utah_obd_proxy")
|
||||
|
||||
def test_unapproved_target_label_source_fails_closed(self) -> None:
|
||||
record = valid_record()
|
||||
record["target_outcome_label_source"] = "inferred_four_class"
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "unapproved"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_first_episode_cannot_be_eligible_returning_target(self) -> None:
|
||||
record = valid_record()
|
||||
record["episode_number"] = 1
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "not a returning"):
|
||||
baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_ineligible_row_is_not_parsed_as_a_supervised_target(self) -> None:
|
||||
record = valid_record()
|
||||
record["eligible_returning_target"] = "false"
|
||||
record["first_outcome"] = ""
|
||||
self.assertIsNone(baselines.episode_from_record(record, 2))
|
||||
|
||||
def test_private_output_path_guard(self) -> None:
|
||||
accepted = PROJECT_ROOT / "artifacts/private/baselines"
|
||||
rejected = PROJECT_ROOT / "artifacts/public/baselines"
|
||||
self.assertEqual(baselines.require_private_path(accepted, "output"), accepted)
|
||||
with self.assertRaisesRegex(baselines.DataValidationError, "must remain"):
|
||||
baselines.require_private_path(rejected, "output")
|
||||
|
||||
|
||||
class MetricTests(unittest.TestCase):
|
||||
def test_perfect_ranking_metrics(self) -> None:
|
||||
targets = [0, 1, 0, 1]
|
||||
probabilities = [0.1, 0.9, 0.2, 0.8]
|
||||
metrics = baselines.binary_metrics(targets, probabilities)
|
||||
self.assertAlmostEqual(metrics["average_precision"], 1.0)
|
||||
self.assertAlmostEqual(metrics["roc_auc"], 1.0)
|
||||
self.assertAlmostEqual(metrics["brier"], 0.025)
|
||||
self.assertAlmostEqual(metrics["top_5_precision"], 1.0)
|
||||
self.assertAlmostEqual(metrics["top_5_capture"], 0.1)
|
||||
|
||||
def test_capacity_metrics_fractionally_handle_boundary_ties(self) -> None:
|
||||
targets = [0, 1] * 50
|
||||
probabilities = [0.2] * 100
|
||||
top_5 = baselines.top_capacity_metrics(targets, probabilities, 0.05)
|
||||
top_10 = baselines.top_capacity_metrics(targets, probabilities, 0.10)
|
||||
self.assertAlmostEqual(top_5["precision"], 0.5)
|
||||
self.assertAlmostEqual(top_5["capture"], 0.05)
|
||||
self.assertAlmostEqual(top_10["precision"], 0.5)
|
||||
self.assertAlmostEqual(top_10["capture"], 0.10)
|
||||
|
||||
def test_constant_predictions_have_half_auc(self) -> None:
|
||||
targets = [0, 1, 0, 1]
|
||||
probabilities = [0.25] * 4
|
||||
self.assertAlmostEqual(baselines.roc_auc(targets, probabilities), 0.5)
|
||||
self.assertAlmostEqual(baselines.average_precision(targets, probabilities), 0.5)
|
||||
|
||||
def test_calibration_bins_cover_each_row_once(self) -> None:
|
||||
targets = [0, 1, 0, 1, 1]
|
||||
probabilities = [0.1, 0.2, 0.3, 0.8, 0.9]
|
||||
bins = baselines.calibration_bins(targets, probabilities, 3)
|
||||
self.assertEqual(sum(item["count"] for item in bins), len(targets))
|
||||
self.assertEqual([item["count"] for item in bins], [2, 2, 1])
|
||||
|
||||
|
||||
class BaselineBehaviorTests(unittest.TestCase):
|
||||
def test_default_iteration_budget_is_hardened(self) -> None:
|
||||
args = baselines.parse_args(["--mart", "data/private/example.parquet"])
|
||||
self.assertEqual(args.max_iter, 5000)
|
||||
|
||||
def _episode(
|
||||
self,
|
||||
prior_outcome: object,
|
||||
target: int = 0,
|
||||
bucket: int = 25,
|
||||
episode_start: str = "2020-06-01T09:00:00",
|
||||
partition: str = "train",
|
||||
source_era: str = "slc",
|
||||
label_source: str = "overall_result",
|
||||
):
|
||||
record = valid_record(
|
||||
episode_start=episode_start,
|
||||
partition=partition,
|
||||
target=target,
|
||||
bucket=bucket,
|
||||
)
|
||||
record["prior_first_outcome"] = prior_outcome
|
||||
record["source_era"] = source_era
|
||||
record["target_outcome_label_source"] = label_source
|
||||
return baselines.episode_from_record(record, 2)
|
||||
|
||||
def test_literal_previous_outcome_and_missing_fallback(self) -> None:
|
||||
rows = [
|
||||
self._episode("pass"),
|
||||
self._episode("fail"),
|
||||
self._episode("reject"),
|
||||
self._episode(""),
|
||||
]
|
||||
probabilities, fallback_count = baselines.literal_previous_probabilities(rows, 0.2)
|
||||
self.assertEqual(probabilities, [0.0, 1.0, 1.0, 0.2])
|
||||
self.assertEqual(fallback_count, 1)
|
||||
|
||||
def test_audit_rows_are_excluded_from_fit_cohort(self) -> None:
|
||||
rows = [self._episode("pass", bucket=5), self._episode("pass", bucket=25)]
|
||||
non_audit = baselines.evaluation_cohorts(rows, include_audit_breakout=False)
|
||||
self.assertEqual(non_audit[0][0], "non_audit")
|
||||
self.assertEqual(len(non_audit[0][1]), 1)
|
||||
self.assertFalse(non_audit[0][1][0].audit_vehicle)
|
||||
|
||||
def test_locked_manifest_counts_withhold_labels_until_unlocked(self) -> None:
|
||||
locked = self._episode(
|
||||
"pass",
|
||||
target=1,
|
||||
bucket=25,
|
||||
episode_start="2025-06-01T09:00:00",
|
||||
partition="test",
|
||||
)
|
||||
locked_proxy = self._episode(
|
||||
"pass",
|
||||
target=0,
|
||||
bucket=26,
|
||||
episode_start="2025-06-01T09:00:00",
|
||||
partition="test",
|
||||
source_era="utah",
|
||||
label_source="utah_obd_proxy",
|
||||
)
|
||||
mart = baselines.MartData(
|
||||
rows_by_partition={
|
||||
"train": [],
|
||||
"tune": [],
|
||||
"calibrate": [],
|
||||
"locked_test": [locked, locked_proxy],
|
||||
},
|
||||
input_rows=2,
|
||||
ineligible_rows=0,
|
||||
eligible_rows=2,
|
||||
)
|
||||
closed, closed_eras, closed_labels = baselines.manifest_audit_counts(
|
||||
mart, evaluate_locked=False
|
||||
)
|
||||
self.assertTrue(closed["locked_test"]["labels_withheld"])
|
||||
self.assertNotIn("nonpass", closed["locked_test"])
|
||||
self.assertNotIn("nonpass", closed_eras["locked_test"]["slc"])
|
||||
self.assertEqual(
|
||||
closed_labels["locked_test"],
|
||||
{"overall_result": 1, "utah_obd_proxy": 1},
|
||||
)
|
||||
|
||||
opened, opened_eras, opened_labels = baselines.manifest_audit_counts(
|
||||
mart, evaluate_locked=True
|
||||
)
|
||||
self.assertEqual(opened["locked_test"]["nonpass"], 1)
|
||||
self.assertEqual(opened_eras["locked_test"]["slc"]["nonpass"], 1)
|
||||
self.assertEqual(
|
||||
opened_labels["locked_test"],
|
||||
{"overall_result": 1, "utah_obd_proxy": 1},
|
||||
)
|
||||
self.assertFalse(
|
||||
baselines.TARGET_CONTRACT["utah_obd_proxy_four_class_approved"]
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(SKLEARN_AVAILABLE, "numpy/scikit-learn are not installed")
|
||||
class ModelingIntegrationTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _partition_records(
|
||||
timestamp: str,
|
||||
partition: str,
|
||||
start_token: int,
|
||||
future_category: bool = False,
|
||||
) -> List[Mapping[str, object]]:
|
||||
records: List[Mapping[str, object]] = []
|
||||
for index in range(40):
|
||||
target = 1 if index % 4 == 0 else 0
|
||||
bucket = index % 100
|
||||
record = valid_record(
|
||||
episode_start=timestamp,
|
||||
partition=partition,
|
||||
target=target,
|
||||
bucket=bucket,
|
||||
token_number=start_token + index,
|
||||
)
|
||||
record["vehicle_age"] = 2 + index % 25
|
||||
record["prior_episode_count"] = 1 + index % 6
|
||||
record["prior_total_attempt_count"] = 1 + index % 10
|
||||
record["prior_attempt_count"] = 1 + index % 3
|
||||
record["days_since_prior_episode"] = 250 + index * 4
|
||||
record["days_since_prior_adverse"] = "" if index % 3 else 500 + index
|
||||
record["prior_nonpass_rate"] = (index % 4) / 4
|
||||
record["prior_first_outcome"] = "fail" if index % 5 == 0 else "pass"
|
||||
if future_category:
|
||||
record["public_county"] = "future_only_county"
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
def _synthetic_mart(self):
|
||||
records: List[Mapping[str, object]] = []
|
||||
records.extend(self._partition_records("2022-06-01", "train", 1))
|
||||
records.extend(self._partition_records("2023-06-01", "tune", 101))
|
||||
records.extend(
|
||||
self._partition_records(
|
||||
"2024-06-01", "calibrate", 201, future_category=True
|
||||
)
|
||||
)
|
||||
records.extend(self._partition_records("2025-06-01", "test", 301))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "mart.csv"
|
||||
write_csv(path, records)
|
||||
return baselines.load_mart(path)
|
||||
|
||||
def test_train_only_preprocessing_and_locked_evaluation_gate(self) -> None:
|
||||
mart = self._synthetic_mart()
|
||||
trained = baselines.train_baselines(
|
||||
mart,
|
||||
c_grid=[0.1],
|
||||
min_category_count=1,
|
||||
seed=7,
|
||||
max_iter=2000,
|
||||
)
|
||||
|
||||
feature_names = set(trained.encoder.vectorizer.get_feature_names_out())
|
||||
self.assertNotIn("cat__public_county=future_only_county", feature_names)
|
||||
self.assertFalse(any("source_era" in name for name in feature_names))
|
||||
self.assertFalse(
|
||||
any("target_outcome_label_source" in name for name in feature_names)
|
||||
)
|
||||
|
||||
locked_metrics, _ = baselines.evaluate_models(
|
||||
mart, trained, evaluate_locked=False, bin_count=5
|
||||
)
|
||||
self.assertNotIn("locked_test", {row["partition"] for row in locked_metrics})
|
||||
|
||||
unlocked_metrics, _ = baselines.evaluate_models(
|
||||
mart, trained, evaluate_locked=True, bin_count=5
|
||||
)
|
||||
locked_rows = [
|
||||
row for row in unlocked_metrics if row["partition"] == "locked_test"
|
||||
]
|
||||
self.assertTrue(locked_rows)
|
||||
self.assertIn("never_fit_audit", {row["cohort"] for row in locked_rows})
|
||||
self.assertIn("logistic_platt", {row["model"] for row in locked_rows})
|
||||
|
||||
tuning = trained.tuning_results[0]
|
||||
self.assertTrue(tuning["converged"])
|
||||
self.assertTrue(tuning["finite_coefficients"])
|
||||
self.assertTrue(tuning["finite_probabilities"])
|
||||
self.assertTrue(tuning["eligible_for_selection"])
|
||||
self.assertLess(tuning["n_iter"], trained.max_iter)
|
||||
self.assertLess(trained.selected_n_iter, trained.max_iter)
|
||||
self.assertLess(trained.platt_n_iter, trained.max_iter)
|
||||
self.assertEqual(trained.platt_model.solver, "liblinear")
|
||||
self.assertEqual(trained.platt_model.penalty, "l2")
|
||||
self.assertEqual(trained.platt_model.C, 1000.0)
|
||||
self.assertEqual(
|
||||
baselines.PLATT_CALIBRATION_CONFIG,
|
||||
{
|
||||
"method": "sigmoid_platt_scaling",
|
||||
"solver": "liblinear",
|
||||
"penalty": "l2",
|
||||
"c": 1000.0,
|
||||
},
|
||||
)
|
||||
|
||||
def test_deliberately_tiny_max_iter_rejects_all_candidates(self) -> None:
|
||||
mart = self._synthetic_mart()
|
||||
with self.assertRaisesRegex(
|
||||
baselines.DataValidationError, "No logistic candidate converged"
|
||||
):
|
||||
baselines.train_baselines(
|
||||
mart,
|
||||
c_grid=[0.03, 0.1],
|
||||
min_category_count=1,
|
||||
seed=7,
|
||||
max_iter=1,
|
||||
)
|
||||
|
||||
def test_nonconverged_and_nonfinite_candidates_are_excluded(self) -> None:
|
||||
import numpy as np
|
||||
import sklearn
|
||||
from sklearn.exceptions import ConvergenceWarning
|
||||
from sklearn.feature_extraction import DictVectorizer
|
||||
|
||||
class ControlledLogistic:
|
||||
def __init__(self, C=1.0, solver="lbfgs", max_iter=100, **_kwargs):
|
||||
self.C = C
|
||||
self.solver = solver
|
||||
self.max_iter = max_iter
|
||||
|
||||
def fit(self, features, _targets):
|
||||
self.n_iter_ = np.asarray([2])
|
||||
self.coef_ = np.zeros((1, features.shape[1]))
|
||||
self.intercept_ = np.zeros(1)
|
||||
if self.solver == "saga" and self.C == 0.03:
|
||||
warnings.warn("controlled nonconvergence", ConvergenceWarning)
|
||||
if self.solver == "saga" and self.C == 0.1:
|
||||
self.coef_[0, 0] = np.nan
|
||||
return self
|
||||
|
||||
def predict_proba(self, features):
|
||||
probability = np.full(features.shape[0], 0.25)
|
||||
return np.column_stack((1.0 - probability, probability))
|
||||
|
||||
def decision_function(self, features):
|
||||
return np.zeros(features.shape[0])
|
||||
|
||||
dependencies = (
|
||||
np,
|
||||
sklearn,
|
||||
DictVectorizer,
|
||||
ControlledLogistic,
|
||||
ConvergenceWarning,
|
||||
)
|
||||
mart = self._synthetic_mart()
|
||||
with mock.patch.object(
|
||||
baselines, "require_ml_dependencies", return_value=dependencies
|
||||
):
|
||||
trained = baselines.train_baselines(
|
||||
mart,
|
||||
c_grid=[0.03, 0.1, 1.0],
|
||||
min_category_count=1,
|
||||
seed=7,
|
||||
max_iter=100,
|
||||
)
|
||||
|
||||
self.assertEqual(trained.selected_c, 1.0)
|
||||
by_c = {result["c"]: result for result in trained.tuning_results}
|
||||
self.assertFalse(by_c[0.03]["converged"])
|
||||
self.assertFalse(by_c[0.03]["eligible_for_selection"])
|
||||
self.assertFalse(by_c[0.1]["finite_coefficients"])
|
||||
self.assertFalse(by_c[0.1]["eligible_for_selection"])
|
||||
self.assertTrue(by_c[1.0]["eligible_for_selection"])
|
||||
|
||||
def test_convergence_capture_replays_unrelated_warnings(self) -> None:
|
||||
from sklearn.exceptions import ConvergenceWarning
|
||||
|
||||
class WarningModel:
|
||||
def fit(self, _features, _targets):
|
||||
warnings.warn("unrelated diagnostic", RuntimeWarning)
|
||||
warnings.warn("did not converge", ConvergenceWarning)
|
||||
|
||||
with self.assertWarnsRegex(RuntimeWarning, "unrelated diagnostic"):
|
||||
messages = baselines.fit_with_convergence_capture(
|
||||
WarningModel(), None, None, ConvergenceWarning
|
||||
)
|
||||
self.assertEqual(messages, ["did not converge"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
564
tests/test_dashboard_export.py
Normal file
564
tests/test_dashboard_export.py
Normal file
@ -0,0 +1,564 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import duckdb
|
||||
|
||||
from scripts import export_dashboard_data as dashboard_export
|
||||
|
||||
|
||||
class DashboardExportTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name)
|
||||
self.mart = self.root / "private_feature_mart.parquet"
|
||||
self.mart_manifest = self.root / "private_feature_mart.parquet.manifest.json"
|
||||
self.model_manifest = self.root / "model_manifest.json"
|
||||
self.model_metrics = self.root / "model_metrics.json"
|
||||
self.public_root = (self.root / "dashboard/public/data").resolve()
|
||||
self._write_mart()
|
||||
self._write_mart_manifest()
|
||||
self._write_model_files()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temporary.cleanup()
|
||||
|
||||
def test_development_mart_is_refused_without_preview_flag(self) -> None:
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=False)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_preview_is_suppressed_deterministic_and_contains_no_locked_data(self) -> None:
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
paths = self._export(development_preview=True)
|
||||
|
||||
expected_names = set(dashboard_export.PUBLIC_ASSET_NAMES) | {
|
||||
dashboard_export.SHA256_MANIFEST_NAME
|
||||
}
|
||||
self.assertEqual(set(paths), expected_names)
|
||||
|
||||
first_bytes = {
|
||||
name: (self.public_root / name).read_bytes()
|
||||
for name in expected_names
|
||||
}
|
||||
joined = b"\n".join(first_bytes.values()).lower()
|
||||
for forbidden in (
|
||||
b"lockedmake",
|
||||
b"secret_locked",
|
||||
b"vehicle_token",
|
||||
b"internal_event_id",
|
||||
b'"vin"',
|
||||
b'"plate"',
|
||||
b'"zip"',
|
||||
b'"station"',
|
||||
b"episode_start",
|
||||
b"target_outcome_label_source",
|
||||
b'"prediction"',
|
||||
):
|
||||
self.assertNotIn(forbidden, joined)
|
||||
|
||||
for name in expected_names:
|
||||
payload = json.loads((self.public_root / name).read_text(encoding="utf-8"))
|
||||
self.assertIs(payload["development_preview"], True)
|
||||
self.assertIs(payload["population_estimate_allowed"], False)
|
||||
|
||||
scorecards = self._asset("cohort_scorecard.json")["rows"]
|
||||
self.assertEqual(
|
||||
scorecards,
|
||||
[
|
||||
{
|
||||
"prior_make": "SAFE",
|
||||
"prior_model": "SUPPORTED",
|
||||
"support_rounded": 400,
|
||||
"nonpass_rate": 0.167,
|
||||
}
|
||||
],
|
||||
)
|
||||
overview = self._asset("overview_period_county.json")["rows"]
|
||||
self.assertEqual(len(overview), 3)
|
||||
self.assertEqual(overview[0]["support_rounded"], 800)
|
||||
self.assertNotIn("n", overview[0])
|
||||
self.assertNotIn("nonpass_count", overview[0])
|
||||
|
||||
diagnostics = self._asset("model_diagnostics.json")["rows"]
|
||||
self.assertEqual(
|
||||
{row["partition"] for row in diagnostics},
|
||||
{"train", "tune", "calibrate"},
|
||||
)
|
||||
self.assertEqual({row["model"] for row in diagnostics}, {"test_model"})
|
||||
self.assertTrue(all("episodes" not in row for row in diagnostics))
|
||||
|
||||
data_manifest = self._asset("data_manifest.json")
|
||||
self.assertEqual(data_manifest["model_versions"], ["test_model_v1"])
|
||||
self.assertRegex(data_manifest["release_id"], r"^[0-9a-f]{64}$")
|
||||
|
||||
checksum_manifest = self._asset(dashboard_export.SHA256_MANIFEST_NAME)
|
||||
for entry in checksum_manifest["files"]:
|
||||
content = (self.public_root / entry["name"]).read_bytes()
|
||||
self.assertEqual(hashlib.sha256(content).hexdigest(), entry["sha256"])
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
self._export(development_preview=True, overwrite=True)
|
||||
second_bytes = {
|
||||
name: (self.public_root / name).read_bytes()
|
||||
for name in expected_names
|
||||
}
|
||||
self.assertEqual(first_bytes, second_bytes)
|
||||
|
||||
def test_unlocked_model_manifest_is_refused(self) -> None:
|
||||
manifest = self._read_json(self.model_manifest)
|
||||
manifest["locked_test_evaluated"] = True
|
||||
self._write_json(self.model_manifest, manifest)
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_locked_metric_row_is_refused_even_when_manifest_is_closed(self) -> None:
|
||||
metrics = self._read_json(self.model_metrics)
|
||||
locked = dict(metrics[0])
|
||||
locked["partition"] = "locked_test"
|
||||
metrics.append(locked)
|
||||
self._write_json(self.model_metrics, metrics)
|
||||
self._refresh_metrics_digest()
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_metrics_digest_mismatch_is_refused(self) -> None:
|
||||
metrics = self._read_json(self.model_metrics)
|
||||
metrics[0]["average_precision"] = 0.99
|
||||
self._write_json(self.model_metrics, metrics)
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_overwrite_refuses_unexpected_file(self) -> None:
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
self._export(development_preview=True)
|
||||
unexpected = self.public_root / "private-notes.txt"
|
||||
unexpected.write_text("must not ride along\n", encoding="utf-8")
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
self._export(development_preview=True, overwrite=True)
|
||||
self.assertTrue(unexpected.exists())
|
||||
|
||||
def test_overwrite_refuses_directory_in_public_contract(self) -> None:
|
||||
self.public_root.mkdir(parents=True)
|
||||
(self.public_root / "data_manifest.json").mkdir()
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
self._export(development_preview=True, overwrite=True)
|
||||
|
||||
def test_pre_2025_locked_partition_aliases_are_refused(self) -> None:
|
||||
for partition in dashboard_export.LOCKED_PARTITION_ALIASES:
|
||||
with self.subTest(partition=partition):
|
||||
self._move_locked_rows_before_boundary(partition)
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_public_validator_rejects_sensitive_keys_and_exact_timestamps(self) -> None:
|
||||
base = {
|
||||
"schema_version": dashboard_export.SCHEMA_VERSION,
|
||||
"development_preview": True,
|
||||
"population_estimate_allowed": False,
|
||||
}
|
||||
unsafe_key = {
|
||||
**base,
|
||||
"rows": [
|
||||
{
|
||||
"year": 2022,
|
||||
"quarter": 1,
|
||||
"public_county": "salt_lake",
|
||||
"support_rounded": 100,
|
||||
"nonpass_rate": 0.2,
|
||||
"vehicle_token": "not-public",
|
||||
}
|
||||
],
|
||||
}
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
dashboard_export.validate_public_asset(
|
||||
"overview_period_county.json", unsafe_key
|
||||
)
|
||||
|
||||
exact_timestamp = {
|
||||
**base,
|
||||
"rows": [
|
||||
{
|
||||
"year": 2022,
|
||||
"quarter": 1,
|
||||
"public_county": "2022-01-01T12:34:56",
|
||||
"support_rounded": 100,
|
||||
"nonpass_rate": 0.2,
|
||||
}
|
||||
],
|
||||
}
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
dashboard_export.validate_public_asset(
|
||||
"overview_period_county.json", exact_timestamp
|
||||
)
|
||||
|
||||
def _export(
|
||||
self,
|
||||
development_preview: bool,
|
||||
overwrite: bool = False,
|
||||
) -> dict:
|
||||
return dashboard_export.export_dashboard_data(
|
||||
mart=self.mart,
|
||||
mart_manifest=self.mart_manifest,
|
||||
bundles=[
|
||||
dashboard_export.ModelBundle(
|
||||
manifest=self.model_manifest,
|
||||
metrics=self.model_metrics,
|
||||
)
|
||||
],
|
||||
output_dir=self.public_root,
|
||||
development_preview=development_preview,
|
||||
overwrite=overwrite,
|
||||
)
|
||||
|
||||
def _write_mart(self) -> None:
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE mart (
|
||||
vehicle_token VARCHAR,
|
||||
is_vin_audit BOOLEAN,
|
||||
episode_number BIGINT,
|
||||
episode_start TIMESTAMP,
|
||||
first_outcome VARCHAR,
|
||||
target_outcome_label_source VARCHAR,
|
||||
target_nonpass INTEGER,
|
||||
eligible_returning_target BOOLEAN,
|
||||
temporal_partition VARCHAR,
|
||||
public_county VARCHAR,
|
||||
source_era VARCHAR,
|
||||
vehicle_age INTEGER,
|
||||
last_observed_make VARCHAR,
|
||||
last_observed_model VARCHAR
|
||||
)
|
||||
"""
|
||||
)
|
||||
rows = []
|
||||
rows.extend(self._group(120, 20, "SAFE", "SUPPORTED"))
|
||||
rows.extend(self._group(99, 19, "LOW", "SUPPORT"))
|
||||
rows.extend(self._group(120, 9, "LOW", "NONPASS"))
|
||||
rows.extend(self._group(120, 111, "LOW", "PASS"))
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
60,
|
||||
"ENTITY",
|
||||
"LOWTOTAL",
|
||||
pass_vehicle_count=40,
|
||||
nonpass_vehicle_count=40,
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
20,
|
||||
"ENTITY",
|
||||
"LOWNONPASS",
|
||||
nonpass_vehicle_count=9,
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
100,
|
||||
"ENTITY",
|
||||
"LOWPASS",
|
||||
pass_vehicle_count=9,
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
20,
|
||||
"SAFE",
|
||||
"SUPPORTED",
|
||||
temporal_partition="tune",
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
20,
|
||||
"SAFE",
|
||||
"SUPPORTED",
|
||||
temporal_partition="calibrate",
|
||||
)
|
||||
)
|
||||
for index in range(120):
|
||||
rows.append(
|
||||
(
|
||||
"secret_vehicle_token",
|
||||
False,
|
||||
2,
|
||||
"2025-01-15 12:34:56",
|
||||
"secret_locked",
|
||||
"secret_locked_source",
|
||||
999,
|
||||
True,
|
||||
"locked_test",
|
||||
"locked_county",
|
||||
"locked_source",
|
||||
5,
|
||||
"LOCKEDMAKE",
|
||||
"LOCKEDMODEL",
|
||||
)
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO mart VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
connection.table("mart").write_parquet(str(self.mart))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
@staticmethod
|
||||
def _group(
|
||||
n: int,
|
||||
nonpass: int,
|
||||
make: str,
|
||||
model: str,
|
||||
temporal_partition: str = "train",
|
||||
pass_vehicle_count: int = None,
|
||||
nonpass_vehicle_count: int = None,
|
||||
) -> list:
|
||||
partition_dates = {
|
||||
"train": "2022-01-15 08:00:00",
|
||||
"tune": "2023-01-15 08:00:00",
|
||||
"calibrate": "2024-01-15 08:00:00",
|
||||
}
|
||||
pass_count = n - nonpass
|
||||
pass_vehicle_count = pass_count if pass_vehicle_count is None else pass_vehicle_count
|
||||
nonpass_vehicle_count = (
|
||||
nonpass if nonpass_vehicle_count is None else nonpass_vehicle_count
|
||||
)
|
||||
rows = []
|
||||
for index in range(n):
|
||||
adverse = index < nonpass
|
||||
class_index = index if adverse else index - nonpass
|
||||
class_vehicle_count = (
|
||||
nonpass_vehicle_count if adverse else pass_vehicle_count
|
||||
)
|
||||
outcome = "nonpass" if adverse else "pass"
|
||||
vehicle_token = "{}_{}_{}_{}_{}".format(
|
||||
make,
|
||||
model,
|
||||
temporal_partition,
|
||||
outcome,
|
||||
class_index % class_vehicle_count,
|
||||
)
|
||||
rows.append(
|
||||
(
|
||||
vehicle_token,
|
||||
False,
|
||||
2,
|
||||
partition_dates[temporal_partition],
|
||||
"fail" if adverse else "pass",
|
||||
"overall_result",
|
||||
1 if adverse else 0,
|
||||
True,
|
||||
temporal_partition,
|
||||
"salt_lake",
|
||||
"slc",
|
||||
5,
|
||||
make,
|
||||
model,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
def _write_mart_manifest(self) -> None:
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
columns = [
|
||||
{"name": row[0], "duckdb_type": row[1]}
|
||||
for row in connection.execute(
|
||||
"DESCRIBE SELECT * FROM read_parquet(?)", [str(self.mart)]
|
||||
).fetchall()
|
||||
]
|
||||
finally:
|
||||
connection.close()
|
||||
self._write_json(
|
||||
self.mart_manifest,
|
||||
{
|
||||
"build_kind": "private_leakage_safe_inspection_feature_mart",
|
||||
"classification": "private_pseudonymized_analytical_mart",
|
||||
"source_data_kind": "vehicle_history_development_sample",
|
||||
"population_estimate_allowed": False,
|
||||
"episode_gap_days": 30,
|
||||
"columns": columns,
|
||||
"parquet_sha256": self._sha256(self.mart),
|
||||
},
|
||||
)
|
||||
|
||||
def _write_model_files(self) -> None:
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
private_support = {
|
||||
partition: (episodes, nonpass, vehicles)
|
||||
for partition, episodes, nonpass, vehicles in connection.execute(
|
||||
"""
|
||||
SELECT temporal_partition,
|
||||
count(*)::BIGINT,
|
||||
sum(target_nonpass)::BIGINT,
|
||||
count(DISTINCT vehicle_token)::BIGINT
|
||||
FROM read_parquet(?)
|
||||
WHERE temporal_partition IN ('train', 'tune', 'calibrate')
|
||||
AND eligible_returning_target
|
||||
AND target_nonpass IN (0, 1)
|
||||
AND NOT is_vin_audit
|
||||
GROUP BY 1
|
||||
""",
|
||||
[str(self.mart)],
|
||||
).fetchall()
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
rows = []
|
||||
for partition, value in (
|
||||
("train", 0.25),
|
||||
("tune", 0.24),
|
||||
("calibrate", 0.23),
|
||||
):
|
||||
episodes, nonpass, vehicles = private_support[partition]
|
||||
rows.append(
|
||||
{
|
||||
"model": "test_model",
|
||||
"partition": partition,
|
||||
"cohort": "non_audit",
|
||||
"episodes": episodes,
|
||||
"nonpass": nonpass,
|
||||
"vehicles": vehicles,
|
||||
"average_precision": value,
|
||||
"brier": 0.1,
|
||||
"log_loss": 0.3,
|
||||
"roc_auc": 0.7,
|
||||
"top_10_capture": 0.2,
|
||||
}
|
||||
)
|
||||
for model, episodes, nonpass in (
|
||||
("low_support", 99, 20),
|
||||
("low_nonpass", 120, 9),
|
||||
("low_pass", 120, 111),
|
||||
("low_vehicles", 120, 20),
|
||||
):
|
||||
rows.append(
|
||||
{
|
||||
"model": model,
|
||||
"partition": "train",
|
||||
"cohort": "non_audit",
|
||||
"episodes": episodes,
|
||||
"nonpass": nonpass,
|
||||
"vehicles": 99 if model == "low_vehicles" else episodes,
|
||||
"average_precision": 0.2,
|
||||
"brier": 0.1,
|
||||
"log_loss": 0.3,
|
||||
"roc_auc": 0.7,
|
||||
"top_10_capture": 0.2,
|
||||
}
|
||||
)
|
||||
self._write_json(self.model_metrics, rows)
|
||||
self._write_json(
|
||||
self.model_manifest,
|
||||
{
|
||||
"classification": "private_model_artifact_no_row_predictions",
|
||||
"input_sha256": self._sha256(self.mart),
|
||||
"metrics_sha256": self._sha256(self.model_metrics),
|
||||
"locked_test_evaluated": False,
|
||||
"model_version": "test_model_v1",
|
||||
},
|
||||
)
|
||||
|
||||
def _refresh_metrics_digest(self) -> None:
|
||||
manifest = self._read_json(self.model_manifest)
|
||||
manifest["metrics_sha256"] = self._sha256(self.model_metrics)
|
||||
self._write_json(self.model_manifest, manifest)
|
||||
|
||||
def _move_locked_rows_before_boundary(self, partition: str) -> None:
|
||||
rewritten = self.root / "rewritten.parquet"
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
connection.execute(
|
||||
"CREATE TABLE rewritten AS SELECT * FROM read_parquet(?)",
|
||||
[str(self.mart)],
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE rewritten
|
||||
SET temporal_partition = ?,
|
||||
episode_start = TIMESTAMP '2024-12-31 12:34:56'
|
||||
WHERE last_observed_make = 'LOCKEDMAKE'
|
||||
""",
|
||||
[partition],
|
||||
)
|
||||
connection.table("rewritten").write_parquet(str(rewritten))
|
||||
finally:
|
||||
connection.close()
|
||||
rewritten.replace(self.mart)
|
||||
|
||||
mart_manifest = self._read_json(self.mart_manifest)
|
||||
mart_manifest["parquet_sha256"] = self._sha256(self.mart)
|
||||
self._write_json(self.mart_manifest, mart_manifest)
|
||||
|
||||
model_manifest = self._read_json(self.model_manifest)
|
||||
model_manifest["input_sha256"] = self._sha256(self.mart)
|
||||
self._write_json(self.model_manifest, model_manifest)
|
||||
|
||||
def _asset(self, name: str) -> object:
|
||||
return self._read_json(self.public_root / name)
|
||||
|
||||
@staticmethod
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, sort_keys=True, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _read_json(path: Path) -> object:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
550
tests/test_feature_mart.py
Normal file
550
tests/test_feature_mart.py
Normal file
@ -0,0 +1,550 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Mapping, Optional
|
||||
|
||||
import duckdb
|
||||
|
||||
from scripts import build_feature_mart as mart
|
||||
|
||||
|
||||
def token(character: str) -> str:
|
||||
return character * 64
|
||||
|
||||
|
||||
def bucket(vehicle_token: str) -> int:
|
||||
return int(vehicle_token[:8], 16) % 100
|
||||
|
||||
|
||||
def event(
|
||||
event_id: int,
|
||||
vehicle_token: str,
|
||||
event_ts: str,
|
||||
outcome: Optional[str],
|
||||
make: Optional[str] = "TOYOTA",
|
||||
model: Optional[str] = "CAMRY",
|
||||
model_year: Optional[int] = 2010,
|
||||
source_era: str = "slc",
|
||||
public_county: str = "salt_lake",
|
||||
label_source: Optional[str] = "overall_result",
|
||||
) -> Dict[str, object]:
|
||||
return {
|
||||
"internal_event_id": event_id,
|
||||
"vehicle_token": vehicle_token,
|
||||
"vehicle_bucket": bucket(vehicle_token),
|
||||
"event_ts": event_ts,
|
||||
"source_era": source_era,
|
||||
"public_county": public_county,
|
||||
"canonical_outcome": outcome,
|
||||
"outcome_label_source": label_source if outcome is not None else None,
|
||||
"program_type": "obd",
|
||||
"test_type": "OBDII",
|
||||
"observed_make": make,
|
||||
"observed_model": model,
|
||||
"observed_model_year": model_year,
|
||||
}
|
||||
|
||||
|
||||
def write_export(
|
||||
directory: Path,
|
||||
name: str,
|
||||
rows: Iterable[Mapping[str, object]],
|
||||
start: str,
|
||||
end: str,
|
||||
kind: str = mart.DEVELOPMENT_KIND,
|
||||
) -> Path:
|
||||
path = directory / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
materialized = list(rows)
|
||||
with gzip.open(path, "wt", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=mart.EXPECTED_COLUMNS)
|
||||
writer.writeheader()
|
||||
writer.writerows(materialized)
|
||||
|
||||
digest = mart.file_sha256(path)
|
||||
manifest = {
|
||||
"export_kind": kind,
|
||||
"query_version": (
|
||||
"inspection_batch_v2"
|
||||
if kind == mart.BOUNDED_KIND
|
||||
else "inspection_history_development_sample_v1"
|
||||
),
|
||||
"query_sha256": "1" * 64,
|
||||
"generated_at_utc": "2026-07-15T12:00:00+00:00",
|
||||
"source_start_inclusive": start,
|
||||
"source_end_exclusive": end,
|
||||
"vehicle_token_key_version": "v1",
|
||||
"vehicle_token_key_fingerprint": "abcdef1234567890",
|
||||
"rows": len(materialized),
|
||||
"columns": list(mart.EXPECTED_COLUMNS),
|
||||
"compressed_file_sha256": digest,
|
||||
"compressed_file_bytes": path.stat().st_size,
|
||||
"classification": (
|
||||
"private_pseudonymized_analytical_staging"
|
||||
if kind == mart.BOUNDED_KIND
|
||||
else "private_pseudonymized_development_only"
|
||||
),
|
||||
}
|
||||
if kind == mart.DEVELOPMENT_KIND:
|
||||
manifest.update(
|
||||
{
|
||||
"population_estimate_allowed": False,
|
||||
"vehicle_sample_limit": 100,
|
||||
"sample_method": "synthetic_test_fixture",
|
||||
}
|
||||
)
|
||||
manifest_path = path.with_name(path.name + ".manifest.json")
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
class FeatureMartTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._temporary_directory.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._temporary_directory.cleanup()
|
||||
|
||||
def build(self, input_path: Path) -> mart.BuildSummary:
|
||||
return mart.build_feature_mart(
|
||||
mart.BuildConfig(
|
||||
input_path=input_path,
|
||||
database_path=self.root / "warehouse.duckdb",
|
||||
output_path=self.root / "feature_mart.parquet",
|
||||
memory_limit="512MB",
|
||||
threads=1,
|
||||
allow_external_output=True,
|
||||
)
|
||||
)
|
||||
|
||||
def row_dict(
|
||||
self, connection: duckdb.DuckDBPyConnection, sql: str, parameters=None
|
||||
) -> Dict[str, object]:
|
||||
cursor = connection.execute(sql, parameters or [])
|
||||
names = [item[0] for item in cursor.description]
|
||||
row = cursor.fetchone()
|
||||
self.assertIsNotNone(row)
|
||||
return dict(zip(names, row))
|
||||
|
||||
def test_episode_boundary_quotes_deduplication_and_prior_only_features(self) -> None:
|
||||
vehicle = token("a")
|
||||
rows = [
|
||||
event(
|
||||
1,
|
||||
vehicle,
|
||||
"2015-01-01 00:00:00",
|
||||
"pass",
|
||||
make="ACME, MOTORS",
|
||||
model='MODEL "Q"',
|
||||
),
|
||||
event(
|
||||
2,
|
||||
vehicle,
|
||||
"2015-01-31 00:00:00",
|
||||
"fail",
|
||||
make="ACME, MOTORS",
|
||||
model='MODEL "Q"',
|
||||
),
|
||||
# Exact analytical duplicate with a distinct source row ID.
|
||||
event(
|
||||
3,
|
||||
vehicle,
|
||||
"2015-01-01 00:00:00",
|
||||
"pass",
|
||||
make="ACME, MOTORS",
|
||||
model='MODEL "Q"',
|
||||
),
|
||||
# Thirty days and one second after the preceding accepted event.
|
||||
event(
|
||||
4,
|
||||
vehicle,
|
||||
"2015-03-02 00:00:01",
|
||||
"reject",
|
||||
make="ACME, MOTORS",
|
||||
model='MODEL "Q"',
|
||||
),
|
||||
event(
|
||||
5,
|
||||
vehicle,
|
||||
"2016-03-02 00:00:01",
|
||||
"fail",
|
||||
make="TARGET, LEAK",
|
||||
model="CURRENT ROW",
|
||||
),
|
||||
event(
|
||||
6,
|
||||
vehicle,
|
||||
"2016-03-03 00:00:01",
|
||||
"pass",
|
||||
make="LATER LEAK",
|
||||
model="LATER ATTEMPT",
|
||||
),
|
||||
]
|
||||
source = write_export(
|
||||
self.root,
|
||||
"history.csv.gz",
|
||||
rows,
|
||||
"2010-01-01T00:00:00",
|
||||
"2020-01-01T00:00:00",
|
||||
)
|
||||
summary = self.build(source)
|
||||
self.assertEqual(summary.clean_events, 5)
|
||||
self.assertEqual(summary.episodes, 3)
|
||||
self.assertFalse(summary.population_estimate_allowed)
|
||||
|
||||
connection = duckdb.connect(str(summary.database_path), read_only=True)
|
||||
try:
|
||||
first_episode = self.row_dict(
|
||||
connection,
|
||||
"""
|
||||
SELECT attempt_count, first_outcome, final_outcome
|
||||
FROM inspection_episodes
|
||||
WHERE vehicle_token = ? AND episode_number = 1
|
||||
""",
|
||||
[vehicle],
|
||||
)
|
||||
self.assertEqual(first_episode["attempt_count"], 2)
|
||||
self.assertEqual(first_episode["first_outcome"], "pass")
|
||||
self.assertEqual(first_episode["final_outcome"], "fail")
|
||||
|
||||
target = self.row_dict(
|
||||
connection,
|
||||
"SELECT * FROM feature_mart WHERE vehicle_token = ? "
|
||||
"AND episode_number = 3",
|
||||
[vehicle],
|
||||
)
|
||||
self.assertEqual(target["target_nonpass"], 1)
|
||||
self.assertEqual(
|
||||
target["target_outcome_label_source"], "overall_result"
|
||||
)
|
||||
self.assertTrue(target["eligible_returning_target"])
|
||||
self.assertEqual(target["prior_episode_count"], 2)
|
||||
self.assertEqual(target["prior_total_attempt_count"], 3)
|
||||
self.assertEqual(target["prior_attempt_count"], 1)
|
||||
self.assertEqual(target["prior_first_outcome"], "reject")
|
||||
self.assertEqual(target["prior_final_outcome"], "reject")
|
||||
self.assertEqual(target["last_observed_make"], "ACME, MOTORS")
|
||||
self.assertEqual(target["last_observed_model"], 'MODEL "Q"')
|
||||
self.assertEqual(target["vehicle_age"], 6)
|
||||
self.assertEqual(target["temporal_partition"], "train")
|
||||
|
||||
names = {
|
||||
row[0]
|
||||
for row in connection.execute(
|
||||
"DESCRIBE SELECT * FROM feature_mart"
|
||||
).fetchall()
|
||||
}
|
||||
required = {
|
||||
"vehicle_token",
|
||||
"vehicle_bucket",
|
||||
"is_vin_audit",
|
||||
"episode_number",
|
||||
"episode_start",
|
||||
"first_outcome",
|
||||
"target_nonpass",
|
||||
"eligible_returning_target",
|
||||
"temporal_partition",
|
||||
"vehicle_age",
|
||||
"prior_episode_count",
|
||||
"prior_total_attempt_count",
|
||||
"prior_attempt_count",
|
||||
"days_since_prior_episode",
|
||||
"days_since_prior_adverse",
|
||||
"prior_nonpass_rate",
|
||||
"public_county",
|
||||
"source_era",
|
||||
"target_season",
|
||||
"prior_first_outcome",
|
||||
"prior_final_outcome",
|
||||
"last_observed_make",
|
||||
"last_observed_model",
|
||||
}
|
||||
self.assertTrue(required.issubset(names))
|
||||
self.assertTrue(
|
||||
{
|
||||
"attempt_count",
|
||||
"final_outcome",
|
||||
"eventually_passed",
|
||||
"episode_end",
|
||||
"obd_result",
|
||||
}.isdisjoint(names)
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def test_conflicting_timestamp_is_quarantined_and_null_first_stays_null(self) -> None:
|
||||
conflicted = token("b")
|
||||
unlabeled = token("c")
|
||||
rows = [
|
||||
event(10, conflicted, "2014-01-01", "pass"),
|
||||
event(11, conflicted, "2016-01-01", "pass"),
|
||||
event(12, conflicted, "2016-01-01", "fail"),
|
||||
event(13, conflicted, "2017-01-01", "pass"),
|
||||
event(20, unlabeled, "2015-01-01", "pass"),
|
||||
event(21, unlabeled, "2016-01-01", None),
|
||||
event(22, unlabeled, "2016-01-02", "pass"),
|
||||
]
|
||||
source = write_export(
|
||||
self.root,
|
||||
"history.csv.gz",
|
||||
rows,
|
||||
"2010-01-01",
|
||||
"2020-01-01",
|
||||
)
|
||||
summary = self.build(source)
|
||||
connection = duckdb.connect(str(summary.database_path), read_only=True)
|
||||
try:
|
||||
conflict_count = connection.execute(
|
||||
"SELECT count(*) FROM event_exclusions "
|
||||
"WHERE exclusion_reason = 'conflicting_same_timestamp'"
|
||||
).fetchone()[0]
|
||||
self.assertEqual(conflict_count, 2)
|
||||
target = self.row_dict(
|
||||
connection,
|
||||
"SELECT first_outcome, target_nonpass, "
|
||||
"eligible_returning_target, eligibility_exclusion_reason "
|
||||
"FROM feature_mart WHERE vehicle_token = ? "
|
||||
"AND episode_start = TIMESTAMP '2016-01-01'",
|
||||
[unlabeled],
|
||||
)
|
||||
self.assertIsNone(target["first_outcome"])
|
||||
self.assertIsNone(target["target_nonpass"])
|
||||
self.assertFalse(target["eligible_returning_target"])
|
||||
self.assertEqual(
|
||||
target["eligibility_exclusion_reason"], "unlabeled_first_outcome"
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def test_bounded_files_are_combined_before_episode_construction(self) -> None:
|
||||
vehicle = token("d")
|
||||
first = write_export(
|
||||
self.root,
|
||||
"inspection_202601.csv.gz",
|
||||
[event(30, vehicle, "2026-01-31 12:00:00", "pass")],
|
||||
"2026-01-01",
|
||||
"2026-02-01",
|
||||
kind=mart.BOUNDED_KIND,
|
||||
)
|
||||
write_export(
|
||||
self.root,
|
||||
"inspection_202602.csv.gz",
|
||||
[event(31, vehicle, "2026-02-01 12:00:00", "fail")],
|
||||
"2026-02-01",
|
||||
"2026-03-01",
|
||||
kind=mart.BOUNDED_KIND,
|
||||
)
|
||||
summary = self.build(first.parent)
|
||||
self.assertTrue(summary.population_estimate_allowed)
|
||||
connection = duckdb.connect(str(summary.database_path), read_only=True)
|
||||
try:
|
||||
result = connection.execute(
|
||||
"SELECT count(*), max(attempt_count) FROM inspection_episodes"
|
||||
).fetchone()
|
||||
self.assertEqual(result, (1, 2))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def test_vehicle_quality_eligibility_uses_only_prior_activity(self) -> None:
|
||||
vehicle = token("e")
|
||||
rows: List[Mapping[str, object]] = [
|
||||
event(40, vehicle, "2014-01-01 00:00:00", "pass"),
|
||||
event(41, vehicle, "2016-01-01 00:00:00", "pass"),
|
||||
]
|
||||
rows.extend(
|
||||
event(42 + index, vehicle, f"2016-03-10 0{index}:00:00", "fail")
|
||||
for index in range(5)
|
||||
)
|
||||
rows.append(event(47, vehicle, "2017-04-15 00:00:00", "pass"))
|
||||
source = write_export(
|
||||
self.root,
|
||||
"history.csv.gz",
|
||||
rows,
|
||||
"2010-01-01",
|
||||
"2020-01-01",
|
||||
)
|
||||
summary = self.build(source)
|
||||
connection = duckdb.connect(str(summary.database_path), read_only=True)
|
||||
try:
|
||||
current_busy_episode = self.row_dict(
|
||||
connection,
|
||||
"SELECT eligible_returning_target, prior_max_events_in_day "
|
||||
"FROM feature_mart WHERE vehicle_token = ? "
|
||||
"AND episode_start = TIMESTAMP '2016-03-10 00:00:00'",
|
||||
[vehicle],
|
||||
)
|
||||
self.assertTrue(current_busy_episode["eligible_returning_target"])
|
||||
self.assertEqual(current_busy_episode["prior_max_events_in_day"], 1)
|
||||
|
||||
after_busy_episode = self.row_dict(
|
||||
connection,
|
||||
"SELECT eligible_returning_target, prior_max_events_in_day, "
|
||||
"eligibility_exclusion_reason FROM feature_mart "
|
||||
"WHERE vehicle_token = ? "
|
||||
"AND episode_start = TIMESTAMP '2017-04-15 00:00:00'",
|
||||
[vehicle],
|
||||
)
|
||||
self.assertFalse(after_busy_episode["eligible_returning_target"])
|
||||
self.assertEqual(after_busy_episode["prior_max_events_in_day"], 5)
|
||||
self.assertEqual(
|
||||
after_busy_episode["eligibility_exclusion_reason"],
|
||||
"prior_daily_activity_over_4",
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def test_digest_mismatch_fails_before_build(self) -> None:
|
||||
source = write_export(
|
||||
self.root,
|
||||
"history.csv.gz",
|
||||
[event(50, token("f"), "2016-01-01", "pass")],
|
||||
"2010-01-01",
|
||||
"2020-01-01",
|
||||
)
|
||||
manifest_path = source.with_name(source.name + ".manifest.json")
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
manifest["compressed_file_sha256"] = "0" * 64
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with self.assertRaisesRegex(mart.MartBuildError, "SHA-256 mismatch"):
|
||||
self.build(source)
|
||||
|
||||
def test_outcome_label_source_is_validated_and_first_row_lineage_is_audit_only(
|
||||
self,
|
||||
) -> None:
|
||||
utah_vehicle = token("7")
|
||||
rows = [
|
||||
event(
|
||||
70,
|
||||
utah_vehicle,
|
||||
"2015-01-01",
|
||||
"pass",
|
||||
source_era="utah",
|
||||
public_county="utah",
|
||||
label_source="utah_obd_proxy",
|
||||
),
|
||||
event(
|
||||
71,
|
||||
utah_vehicle,
|
||||
"2016-01-01",
|
||||
"fail",
|
||||
source_era="utah",
|
||||
public_county="utah",
|
||||
label_source="utah_obd_proxy",
|
||||
),
|
||||
event(
|
||||
72,
|
||||
token("8"),
|
||||
"2016-01-01",
|
||||
"pass",
|
||||
source_era="slc",
|
||||
public_county="salt_lake",
|
||||
label_source="utah_obd_proxy",
|
||||
),
|
||||
event(
|
||||
73,
|
||||
token("9"),
|
||||
"2016-01-01",
|
||||
"pass",
|
||||
label_source=None,
|
||||
),
|
||||
event(
|
||||
74,
|
||||
token("0"),
|
||||
"2016-01-01",
|
||||
"pass",
|
||||
label_source="unknown_source",
|
||||
),
|
||||
]
|
||||
source = write_export(
|
||||
self.root,
|
||||
"history.csv.gz",
|
||||
rows,
|
||||
"2010-01-01",
|
||||
"2020-01-01",
|
||||
)
|
||||
summary = self.build(source)
|
||||
connection = duckdb.connect(str(summary.database_path), read_only=True)
|
||||
try:
|
||||
target = self.row_dict(
|
||||
connection,
|
||||
"SELECT target_outcome_label_source, source_era "
|
||||
"FROM feature_mart WHERE vehicle_token = ? "
|
||||
"AND episode_number = 2",
|
||||
[utah_vehicle],
|
||||
)
|
||||
self.assertEqual(
|
||||
target["target_outcome_label_source"], "utah_obd_proxy"
|
||||
)
|
||||
self.assertEqual(target["source_era"], "utah")
|
||||
|
||||
reasons = dict(
|
||||
connection.execute(
|
||||
"SELECT exclusion_reason, count(*) FROM event_exclusions "
|
||||
"GROUP BY exclusion_reason"
|
||||
).fetchall()
|
||||
)
|
||||
self.assertEqual(reasons["utah_proxy_non_utah_source"], 1)
|
||||
self.assertEqual(reasons["missing_outcome_label_source"], 1)
|
||||
self.assertEqual(reasons["invalid_outcome_label_source"], 1)
|
||||
|
||||
schema_names = {
|
||||
row[0]
|
||||
for row in connection.execute(
|
||||
"DESCRIBE SELECT * FROM feature_mart"
|
||||
).fetchall()
|
||||
}
|
||||
self.assertIn("target_outcome_label_source", schema_names)
|
||||
self.assertNotIn("outcome_label_source", schema_names)
|
||||
self.assertNotIn("obd_result", schema_names)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def test_bounded_gap_fails_closed(self) -> None:
|
||||
vehicle = token("1")
|
||||
write_export(
|
||||
self.root,
|
||||
"one.csv.gz",
|
||||
[event(60, vehicle, "2026-01-15", "pass")],
|
||||
"2026-01-01",
|
||||
"2026-02-01",
|
||||
kind=mart.BOUNDED_KIND,
|
||||
)
|
||||
write_export(
|
||||
self.root,
|
||||
"two.csv.gz",
|
||||
[event(61, vehicle, "2026-03-15", "pass")],
|
||||
"2026-03-01",
|
||||
"2026-04-01",
|
||||
kind=mart.BOUNDED_KIND,
|
||||
)
|
||||
with self.assertRaisesRegex(mart.MartBuildError, "interval gap"):
|
||||
self.build(self.root)
|
||||
|
||||
incomplete = mart.build_feature_mart(
|
||||
mart.BuildConfig(
|
||||
input_path=self.root,
|
||||
database_path=self.root / "incomplete.duckdb",
|
||||
output_path=self.root / "incomplete.parquet",
|
||||
memory_limit="512MB",
|
||||
threads=1,
|
||||
allow_gaps=True,
|
||||
allow_external_output=True,
|
||||
)
|
||||
)
|
||||
self.assertFalse(incomplete.population_estimate_allowed)
|
||||
build_manifest = json.loads(
|
||||
incomplete.manifest_path.read_text(encoding="utf-8")
|
||||
)
|
||||
self.assertFalse(build_manifest["population_estimate_allowed"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
299
tests/test_private_export.py
Normal file
299
tests/test_private_export.py
Normal file
@ -0,0 +1,299 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from scripts import export_history_sample as history_export
|
||||
from scripts import export_inspection_batch as exporter
|
||||
|
||||
|
||||
VALID_VIN = "1ABCD23EFGH456789"
|
||||
TEST_KEY = bytes(range(32))
|
||||
|
||||
|
||||
class PrivateExportTests(unittest.TestCase):
|
||||
def test_normalize_vin_canonicalizes_case_and_whitespace(self) -> None:
|
||||
self.assertEqual(
|
||||
exporter.normalize_vin(f" {VALID_VIN.lower()}\n"),
|
||||
VALID_VIN,
|
||||
)
|
||||
|
||||
def test_normalize_vin_rejects_invalid_values_and_placeholders(self) -> None:
|
||||
invalid_vins = (
|
||||
"",
|
||||
"1ABCD23EFGH45678",
|
||||
"1ABCD23EFGH4567890",
|
||||
"1ABCD23EFGI456789",
|
||||
"11111111111111111",
|
||||
"AAAAAAAAAAAAAAAAA",
|
||||
"12345678901234567",
|
||||
"98765432109876543",
|
||||
)
|
||||
for vin in invalid_vins:
|
||||
with self.subTest(vin=vin):
|
||||
self.assertIsNone(exporter.normalize_vin(vin))
|
||||
|
||||
def test_hmac_is_deterministic_and_version_domain_separated(self) -> None:
|
||||
expected_v1 = hmac.new(
|
||||
TEST_KEY,
|
||||
b"utah-vehicle-health/v1/vin\0" + VALID_VIN.encode("ascii"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
token_v1 = exporter.vehicle_token(VALID_VIN, TEST_KEY, "v1")
|
||||
repeated_v1 = exporter.vehicle_token(VALID_VIN, TEST_KEY, "v1")
|
||||
token_v2 = exporter.vehicle_token(VALID_VIN, TEST_KEY, "v2")
|
||||
|
||||
self.assertEqual(token_v1, expected_v1)
|
||||
self.assertEqual(repeated_v1, token_v1)
|
||||
self.assertNotEqual(token_v2, token_v1)
|
||||
self.assertEqual(len(token_v1), 64)
|
||||
|
||||
def test_recognized_overall_result_precedes_utah_obd_proxy(self) -> None:
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
"PASS", "FAIL", "utah", "other", "C"
|
||||
),
|
||||
("pass", "overall_result"),
|
||||
)
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
" f ", "PASS", " UTAH ", "tsi", "TSI"
|
||||
),
|
||||
("fail", "overall_result"),
|
||||
)
|
||||
|
||||
def test_utah_only_obd_fallback_uses_recognized_values(self) -> None:
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
"", " P ", "utah", " ObD ", " obd "
|
||||
),
|
||||
("pass", "utah_obd_proxy"),
|
||||
)
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
"B", "REJECT", "UTAH", "obd", "OBD"
|
||||
),
|
||||
("reject", "utah_obd_proxy"),
|
||||
)
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
None, "FAIL", "weber", "obd", "OBD"
|
||||
),
|
||||
(None, None),
|
||||
)
|
||||
|
||||
def test_utah_non_obd_program_or_test_rows_do_not_use_proxy(self) -> None:
|
||||
excluded_cases = (
|
||||
("other", "C"),
|
||||
("tsi", "TSI"),
|
||||
("obd", "C"),
|
||||
("other", "OBD"),
|
||||
("", ""),
|
||||
(None, None),
|
||||
)
|
||||
for program_type, test_type in excluded_cases:
|
||||
with self.subTest(program_type=program_type, test_type=test_type):
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
"B", "PASS", "utah", program_type, test_type
|
||||
),
|
||||
(None, None),
|
||||
)
|
||||
|
||||
def test_unknown_blank_and_b_proxy_values_remain_unlabeled(self) -> None:
|
||||
cases = (
|
||||
(None, None, "utah", "obd", "OBD"),
|
||||
("", "", "utah", "obd", "OBD"),
|
||||
("B", "B", "utah", "obd", "OBD"),
|
||||
("UNKNOWN", "PASS", "slco", "obd", "OBD"),
|
||||
)
|
||||
for overall, obd, source, program_type, test_type in cases:
|
||||
with self.subTest(
|
||||
overall=overall,
|
||||
obd=obd,
|
||||
source=source,
|
||||
program_type=program_type,
|
||||
test_type=test_type,
|
||||
):
|
||||
self.assertEqual(
|
||||
exporter.canonicalize_outcome(
|
||||
overall, obd, source, program_type, test_type
|
||||
),
|
||||
(None, None),
|
||||
)
|
||||
|
||||
def test_hmac_configuration_rejects_malformed_or_weak_keys(self) -> None:
|
||||
invalid_keys = (
|
||||
"",
|
||||
"not-hex",
|
||||
"00" * 31,
|
||||
"00" * 33,
|
||||
"00" * 32,
|
||||
"ab" * 32,
|
||||
)
|
||||
for encoded_key in invalid_keys:
|
||||
with self.subTest(encoded_key=encoded_key[:12]):
|
||||
with self._hmac_environment(encoded_key):
|
||||
with self.assertRaises(ValueError):
|
||||
exporter.get_hmac_configuration()
|
||||
|
||||
def test_hmac_configuration_accepts_exact_random_key(self) -> None:
|
||||
with self._hmac_environment(TEST_KEY.hex(), version="key-2026.1"):
|
||||
key, version, fingerprint = exporter.get_hmac_configuration()
|
||||
|
||||
self.assertEqual(key, TEST_KEY)
|
||||
self.assertEqual(version, "key-2026.1")
|
||||
self.assertRegex(fingerprint, r"^[0-9a-f]{16}$")
|
||||
|
||||
def test_vehicle_bucket_is_stable_after_vin_normalization(self) -> None:
|
||||
row = self._source_row(VALID_VIN)
|
||||
normalized_row = exporter.private_row(row, TEST_KEY, "v1")
|
||||
lower_row = exporter.private_row(
|
||||
self._source_row(f" {VALID_VIN.lower()} "),
|
||||
TEST_KEY,
|
||||
"v1",
|
||||
)
|
||||
|
||||
self.assertIsNotNone(normalized_row)
|
||||
self.assertIsNotNone(lower_row)
|
||||
assert normalized_row is not None
|
||||
assert lower_row is not None
|
||||
token = normalized_row[1]
|
||||
bucket = normalized_row[2]
|
||||
self.assertEqual(lower_row[1], token)
|
||||
self.assertEqual(lower_row[2], bucket)
|
||||
self.assertEqual(bucket, int(str(token)[:8], 16) % 100)
|
||||
self.assertIn(bucket, range(100))
|
||||
|
||||
def test_validate_args_enforces_private_output_root(self) -> None:
|
||||
private_output = (
|
||||
exporter.PRIVATE_DATA_ROOT
|
||||
/ "inspection_batches"
|
||||
/ f"unit-{uuid.uuid4().hex}.csv.gz"
|
||||
)
|
||||
validated = exporter.validate_args(self._args(private_output))
|
||||
self.assertEqual(validated, private_output.resolve())
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
external_output = Path(temporary_directory) / "private.csv.gz"
|
||||
with self.assertRaises(ValueError):
|
||||
exporter.validate_args(self._args(external_output))
|
||||
|
||||
allowed = exporter.validate_args(
|
||||
self._args(external_output, allow_external_output=True)
|
||||
)
|
||||
self.assertEqual(allowed, external_output.resolve())
|
||||
|
||||
def test_declared_output_omits_raw_vin_and_raw_obd_result(self) -> None:
|
||||
self.assertNotIn("vin", exporter.OUTPUT_COLUMNS)
|
||||
self.assertNotIn("raw_vin", exporter.OUTPUT_COLUMNS)
|
||||
self.assertNotIn("obd_result", exporter.OUTPUT_COLUMNS)
|
||||
self.assertNotIn("overall_result", exporter.OUTPUT_COLUMNS)
|
||||
self.assertIn("vehicle_token", exporter.OUTPUT_COLUMNS)
|
||||
self.assertIn("outcome_label_source", exporter.OUTPUT_COLUMNS)
|
||||
|
||||
transformed = exporter.private_row(
|
||||
self._source_row(VALID_VIN), TEST_KEY, "v1"
|
||||
)
|
||||
self.assertIsNotNone(transformed)
|
||||
assert transformed is not None
|
||||
self.assertEqual(len(transformed), len(exporter.OUTPUT_COLUMNS))
|
||||
self.assertEqual(
|
||||
transformed[exporter.OUTPUT_COLUMNS.index("outcome_label_source")],
|
||||
"overall_result",
|
||||
)
|
||||
self.assertNotIn(VALID_VIN, transformed)
|
||||
|
||||
def test_both_exporters_share_the_versioned_label_sql(self) -> None:
|
||||
history_sql = history_export.make_sql(0.5, 20260715)
|
||||
|
||||
self.assertIn(exporter.OUTCOME_SELECT_SQL, exporter.EXTRACT_SQL)
|
||||
self.assertIn(exporter.OUTCOME_SELECT_SQL, history_sql)
|
||||
self.assertIn("s.obd_result", exporter.OUTCOME_SELECT_SQL)
|
||||
self.assertIn(
|
||||
"lower(btrim(s.program_type)) = 'obd'",
|
||||
exporter.OUTCOME_SELECT_SQL,
|
||||
)
|
||||
self.assertIn(
|
||||
"upper(btrim(s.test_type)) = 'OBD'",
|
||||
exporter.OUTCOME_SELECT_SQL,
|
||||
)
|
||||
self.assertNotIn("WHEN 'B'", exporter.OUTCOME_SELECT_SQL)
|
||||
self.assertEqual(exporter.QUERY_VERSION, "inspection_batch_v4")
|
||||
self.assertEqual(
|
||||
history_export.QUERY_VERSION,
|
||||
"inspection_history_development_sample_v3",
|
||||
)
|
||||
|
||||
def test_feasibility_sql_uses_labeled_utah_only_proxy(self) -> None:
|
||||
sql_path = exporter.PROJECT_ROOT / "sql/10_episode_cohort_feasibility.sql"
|
||||
sql = sql_path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("lower(btrim(s.county)) = 'utah'", sql)
|
||||
self.assertIn("lower(btrim(s.program_type)) = 'obd'", sql)
|
||||
self.assertIn("upper(btrim(s.test_type)) = 'OBD'", sql)
|
||||
self.assertIn("upper(btrim(s.obd_result))", sql)
|
||||
self.assertIn("AS outcome_label_source", sql)
|
||||
self.assertIn("'utah_obd_proxy'", sql)
|
||||
self.assertNotIn("WHEN 'B' THEN", sql)
|
||||
|
||||
@staticmethod
|
||||
def _source_row(vin: str) -> tuple[object, ...]:
|
||||
return (
|
||||
123,
|
||||
vin,
|
||||
datetime(2025, 1, 15, 12, 0),
|
||||
"slco",
|
||||
"salt_lake",
|
||||
"pass",
|
||||
"overall_result",
|
||||
"obd",
|
||||
"INITIAL",
|
||||
"EXAMPLE",
|
||||
"MODEL",
|
||||
2020,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _args(
|
||||
output: Path, allow_external_output: bool = False
|
||||
) -> SimpleNamespace:
|
||||
start = datetime(2025, 1, 1)
|
||||
return SimpleNamespace(
|
||||
start=start,
|
||||
end=start + timedelta(days=1),
|
||||
output=output,
|
||||
page_size=100,
|
||||
overwrite=False,
|
||||
allow_external_output=allow_external_output,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _hmac_environment(
|
||||
encoded_key: str, version: str = "v1"
|
||||
) -> mock._patch_dict:
|
||||
missing_key_file = Path(tempfile.gettempdir()) / (
|
||||
f"uvh-missing-key-{uuid.uuid4().hex}"
|
||||
)
|
||||
return mock.patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"VIN_HASH_KEY": encoded_key,
|
||||
"VIN_HASH_KEY_FILE": str(missing_key_file),
|
||||
"VIN_HASH_KEY_VERSION": version,
|
||||
},
|
||||
clear=False,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
261
tests/test_tree_model.py
Normal file
261
tests/test_tree_model.py
Normal file
@ -0,0 +1,261 @@
|
||||
"""Tests for the leakage-safe nonlinear candidate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import math
|
||||
import sys
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE_PATH = PROJECT_ROOT / "scripts/train_tree_model.py"
|
||||
SPEC = importlib.util.spec_from_file_location("train_tree_model", MODULE_PATH)
|
||||
if SPEC is None or SPEC.loader is None: # pragma: no cover
|
||||
raise RuntimeError("Could not load scripts/train_tree_model.py")
|
||||
tree = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = tree
|
||||
SPEC.loader.exec_module(tree)
|
||||
baselines = tree.baselines
|
||||
|
||||
SKLEARN_AVAILABLE = (
|
||||
importlib.util.find_spec("numpy") is not None
|
||||
and importlib.util.find_spec("sklearn") is not None
|
||||
)
|
||||
|
||||
|
||||
def episode(
|
||||
number: int,
|
||||
partition: str,
|
||||
year: int,
|
||||
target: int,
|
||||
age: float,
|
||||
make: str,
|
||||
county: str = "salt_lake",
|
||||
audit: bool = False,
|
||||
) -> baselines.EpisodeRow:
|
||||
bucket = 5 if audit else 25 + (number % 70)
|
||||
numeric: Dict[str, float] = {
|
||||
"vehicle_age": age,
|
||||
"prior_episode_count": 1.0 + number % 8,
|
||||
"prior_total_attempt_count": 1.0 + number % 12,
|
||||
"prior_attempt_count": 1.0 + number % 3,
|
||||
"days_since_prior_episode": 300.0 + number % 150,
|
||||
"days_since_prior_adverse": 200.0 + number % 500,
|
||||
"prior_nonpass_rate": (number % 10) / 10.0,
|
||||
}
|
||||
categorical = {
|
||||
"public_county": county,
|
||||
"target_season": ("summer" if number % 2 else "winter"),
|
||||
"prior_first_outcome": ("fail" if number % 4 == 0 else "pass"),
|
||||
"prior_final_outcome": ("fail" if number % 5 == 0 else "pass"),
|
||||
"last_observed_make": make,
|
||||
"last_observed_model": ("model_a" if number % 3 else "model_b"),
|
||||
}
|
||||
return baselines.EpisodeRow(
|
||||
vehicle_token="{:064x}".format(number + year * 100_000),
|
||||
vehicle_bucket=bucket,
|
||||
is_vin_audit=audit,
|
||||
episode_number=2 + number % 10,
|
||||
episode_start=datetime(year, 6, 1),
|
||||
partition=partition,
|
||||
target=target,
|
||||
source_era="slc",
|
||||
target_outcome_label_source="overall_result",
|
||||
prior_first_outcome=categorical["prior_first_outcome"],
|
||||
numeric=numeric,
|
||||
categorical=categorical,
|
||||
)
|
||||
|
||||
|
||||
def partition_rows(partition: str, year: int, count: int, offset: int) -> List[object]:
|
||||
rows: List[object] = []
|
||||
for index in range(count):
|
||||
number = offset + index
|
||||
audit = index % 10 == 0
|
||||
age = float((number * 7) % 30)
|
||||
county = "utah" if number % 2 else "salt_lake"
|
||||
signal = (age > 14.0) != (county == "utah")
|
||||
target = int(signal)
|
||||
if number % 17 == 0:
|
||||
target = 1 - target
|
||||
if audit:
|
||||
make = "audit_only_make"
|
||||
elif partition == "train":
|
||||
make = ("toyota" if number % 3 else "ford")
|
||||
else:
|
||||
make = ("future_make" if number % 7 == 0 else "toyota")
|
||||
rows.append(
|
||||
episode(
|
||||
number=number,
|
||||
partition=partition,
|
||||
year=year,
|
||||
target=target,
|
||||
age=age,
|
||||
make=make,
|
||||
county=county,
|
||||
audit=audit,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def synthetic_mart() -> baselines.MartData:
|
||||
rows_by_partition = {
|
||||
"train": partition_rows("train", 2022, 360, 0),
|
||||
"tune": partition_rows("tune", 2023, 140, 10_000),
|
||||
"calibrate": partition_rows("calibrate", 2024, 140, 20_000),
|
||||
"locked_test": partition_rows("locked_test", 2025, 140, 30_000),
|
||||
}
|
||||
total = sum(len(rows) for rows in rows_by_partition.values())
|
||||
return baselines.MartData(
|
||||
rows_by_partition=rows_by_partition,
|
||||
input_rows=total,
|
||||
ineligible_rows=0,
|
||||
eligible_rows=total,
|
||||
)
|
||||
|
||||
|
||||
class TreeEncoderTests(unittest.TestCase):
|
||||
def test_future_categories_and_values_do_not_change_training_preprocessing(self) -> None:
|
||||
train_rows = [
|
||||
episode(1, "train", 2022, 0, 10.0, "toyota"),
|
||||
episode(2, "train", 2022, 1, 20.0, "toyota"),
|
||||
episode(3, "train", 2022, 0, 30.0, "ford"),
|
||||
]
|
||||
future = episode(4, "tune", 2023, 1, 9_999.0, "future_make")
|
||||
encoder = tree.TreeFeatureEncoder.fit(
|
||||
train_rows, min_category_count=2, max_categories=16
|
||||
)
|
||||
|
||||
self.assertEqual(encoder.numeric_medians["vehicle_age"], 20.0)
|
||||
self.assertEqual(
|
||||
encoder.category_code("last_observed_make", "toyota"),
|
||||
tree.FIRST_KNOWN_CATEGORY_CODE,
|
||||
)
|
||||
self.assertEqual(
|
||||
encoder.category_code("last_observed_make", "ford"),
|
||||
tree.RARE_CATEGORY_CODE,
|
||||
)
|
||||
self.assertEqual(
|
||||
encoder.category_code("last_observed_make", "future_make"),
|
||||
tree.UNKNOWN_CATEGORY_CODE,
|
||||
)
|
||||
self.assertNotIn(
|
||||
"future_make", encoder.seen_categories["last_observed_make"]
|
||||
)
|
||||
self.assertEqual(encoder.numeric_medians["vehicle_age"], 20.0)
|
||||
self.assertTrue(set(tree.EXCLUDED_FROM_PREDICTORS).isdisjoint(
|
||||
encoder.feature_names
|
||||
))
|
||||
self.assertNotIn("source_era", encoder.feature_names)
|
||||
self.assertNotIn("target_outcome_label_source", encoder.feature_names)
|
||||
|
||||
if SKLEARN_AVAILABLE:
|
||||
import numpy as np
|
||||
|
||||
matrix = encoder.transform([future], np)
|
||||
category_index = 2 * len(tree.NUMERIC_FEATURES) + list(
|
||||
tree.CATEGORICAL_FEATURES
|
||||
).index("last_observed_make")
|
||||
self.assertEqual(
|
||||
matrix[0, category_index], float(tree.UNKNOWN_CATEGORY_CODE)
|
||||
)
|
||||
self.assertTrue(np.isfinite(matrix).all())
|
||||
|
||||
@unittest.skipUnless(SKLEARN_AVAILABLE, "scikit-learn is not installed")
|
||||
def test_convergence_gate_rejects_iteration_limit_and_nonfinite_scores(self) -> None:
|
||||
import numpy as np
|
||||
|
||||
at_limit = SimpleNamespace(
|
||||
n_iter_=100,
|
||||
train_score_=np.asarray([-1.0]),
|
||||
validation_score_=np.asarray([-1.0]),
|
||||
)
|
||||
converged, n_iter = tree._hist_converged(at_limit, [], 100, np)
|
||||
self.assertFalse(converged)
|
||||
self.assertEqual(n_iter, 100)
|
||||
|
||||
nonfinite = SimpleNamespace(
|
||||
n_iter_=20,
|
||||
train_score_=np.asarray([-1.0, math.nan]),
|
||||
validation_score_=np.asarray([-1.0]),
|
||||
)
|
||||
converged, _ = tree._hist_converged(nonfinite, [], 100, np)
|
||||
self.assertFalse(converged)
|
||||
|
||||
|
||||
@unittest.skipUnless(SKLEARN_AVAILABLE, "scikit-learn is not installed")
|
||||
class TreeTrainingTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.mart = synthetic_mart()
|
||||
cls.trained = tree.train_tree_model(
|
||||
mart=cls.mart,
|
||||
candidates=[tree.TreeCandidate(0.1, 15, 1.0)],
|
||||
min_category_count=3,
|
||||
max_categories=32,
|
||||
min_samples_leaf=10,
|
||||
max_iter=150,
|
||||
n_iter_no_change=8,
|
||||
calibration_max_iter=1000,
|
||||
seed=20260715,
|
||||
)
|
||||
|
||||
def test_fit_excludes_audit_and_future_categories(self) -> None:
|
||||
seen = self.trained.encoder.seen_categories["last_observed_make"]
|
||||
self.assertNotIn("audit_only_make", seen)
|
||||
self.assertNotIn("future_make", seen)
|
||||
self.assertLess(self.trained.model_n_iter, self.trained.max_iter)
|
||||
self.assertTrue(
|
||||
all(item["eligible_for_selection"] for item in self.trained.tuning_results)
|
||||
)
|
||||
|
||||
def test_locked_metrics_require_explicit_gate(self) -> None:
|
||||
closed_metrics, _closed_bins = tree.evaluate_tree_model(
|
||||
self.mart,
|
||||
self.trained,
|
||||
evaluate_locked=False,
|
||||
bin_count=5,
|
||||
)
|
||||
self.assertNotIn(
|
||||
"locked_test", {row["partition"] for row in closed_metrics}
|
||||
)
|
||||
|
||||
open_metrics, _open_bins = tree.evaluate_tree_model(
|
||||
self.mart,
|
||||
self.trained,
|
||||
evaluate_locked=True,
|
||||
bin_count=5,
|
||||
)
|
||||
locked = [
|
||||
row for row in open_metrics if row["partition"] == "locked_test"
|
||||
]
|
||||
self.assertTrue(locked)
|
||||
self.assertIn("never_fit_audit", {row["cohort"] for row in locked})
|
||||
|
||||
def test_probabilities_are_finite(self) -> None:
|
||||
probabilities = tree.tree_probabilities(
|
||||
self.trained,
|
||||
self.mart.rows_by_partition["calibrate"],
|
||||
calibrated=True,
|
||||
)
|
||||
self.assertTrue(probabilities)
|
||||
self.assertTrue(all(math.isfinite(value) for value in probabilities))
|
||||
self.assertTrue(all(0.0 <= value <= 1.0 for value in probabilities))
|
||||
|
||||
def test_cli_locked_flag_defaults_closed(self) -> None:
|
||||
closed = tree.parse_args(["--mart", "data/private/mart.parquet"])
|
||||
opened = tree.parse_args(
|
||||
["--mart", "data/private/mart.parquet", "--evaluate-locked"]
|
||||
)
|
||||
self.assertFalse(closed.evaluate_locked)
|
||||
self.assertTrue(opened.evaluate_locked)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue
Block a user