repository is now a presentation-ready development prototype
This commit is contained in:
+74
-105
@@ -1,115 +1,84 @@
|
||||
graph TD
|
||||
%% Data Sources
|
||||
PG[(PostgreSQL<br/>countydata)]
|
||||
DMV[("DMV Data<br/>~18M records<br/>2011-2024")]
|
||||
INSP[("Inspection Data<br/>~19.3M records<br/>2010-2026")]
|
||||
flowchart TD
|
||||
SOURCE[(Private inspection histories<br/>read-only source)]
|
||||
SAMPLE[Private page-sampled histories<br/>up to 10,000 vehicles<br/>keyed vehicle tokens]
|
||||
SOURCE -->|read-only bounded export| SAMPLE
|
||||
|
||||
PG --- DMV
|
||||
PG --- INSP
|
||||
SAMPLE --> STAGE[Normalize labels<br/>deduplicate and validate]
|
||||
STAGE --> EPISODES[Build 30-day episodes<br/>first attempt is the target]
|
||||
EPISODES --> MART[Point-in-time feature mart<br/>returning vehicles only]
|
||||
|
||||
%% Extraction
|
||||
PG -->|"export_history_sample.py<br/>(VIN hashing, page sampling)"| CSV["history_sample_10000.csv.gz<br/>+ manifest"]
|
||||
|
||||
%% Feature Mart Build
|
||||
CSV -->|"build_feature_mart.py"| DUCKDB
|
||||
|
||||
subgraph DuckDB["DuckDB Local Warehouse"]
|
||||
direction TB
|
||||
SQL20["20_stage_events.sql<br/>Normalize & validate"]
|
||||
SQL21["21_build_episodes.sql<br/>Group episodes (30-day gap)"]
|
||||
SQL22["22_build_features.sql<br/>Point-in-time feature windows"]
|
||||
SQL20 --> SQL21 --> SQL22
|
||||
end
|
||||
|
||||
DUCKDB --> MART["inspection_feature_mart.parquet<br/>(episode × features, temporal partitions)"]
|
||||
|
||||
%% Temporal Partitions
|
||||
subgraph Partitions["Temporal Splits"]
|
||||
TRAIN["Train: 2016–2022"]
|
||||
TUNE["Tune: 2023"]
|
||||
CAL["Calibrate: 2024"]
|
||||
TEST["Locked Test: 2025"]
|
||||
DRIFT["Shadow Drift: 2026"]
|
||||
end
|
||||
MART -.- Partitions
|
||||
|
||||
%% Model Training
|
||||
MART -->|"train_baselines.py"| BL["artifacts/baselines/baseline_v1/<br/>Logistic Regression +<br/>Prevalence & Prior-Outcome"]
|
||||
MART -->|"train_tree_model.py"| TREE["artifacts/tree/hist_gradient_boosting_v1/<br/>Histogram Gradient Boosting"]
|
||||
|
||||
%% Model Outputs
|
||||
BL --> ARTIFACTS["Model Artifacts<br/>manifest.json, metrics.json,<br/>model.joblib, calibration_bins.csv"]
|
||||
TREE --> ARTIFACTS
|
||||
|
||||
%% Dashboard Export
|
||||
MART -->|"export_dashboard_data.py<br/>(privacy suppression, aggregation)"| JSON
|
||||
|
||||
subgraph JSON["dashboard/public/data/"]
|
||||
direction TB
|
||||
DM["data_manifest.json"]
|
||||
OV["overview_period_county.json"]
|
||||
CS["cohort_scorecard.json"]
|
||||
ARC["age_risk_curve.json"]
|
||||
CQ["coverage_quality.json"]
|
||||
FC["filter_catalog.json"]
|
||||
MD["model_diagnostics.json"]
|
||||
end
|
||||
|
||||
ARTIFACTS -.->|metrics & manifest| JSON
|
||||
|
||||
%% Dashboard Frontend
|
||||
JSON --> SERVER["server.mjs<br/>Node.js HTTP server"]
|
||||
|
||||
subgraph Dashboard["Browser SPA"]
|
||||
direction TB
|
||||
APP["app.js<br/>Routing & pages"]
|
||||
CHARTS["charts.js<br/>D3 visualizations"]
|
||||
DATA["data.js<br/>Data loading & filters"]
|
||||
APP --- CHARTS
|
||||
APP --- DATA
|
||||
end
|
||||
|
||||
SERVER --> Dashboard
|
||||
|
||||
subgraph Pages["Dashboard Pages"]
|
||||
P1["Overview<br/>KPIs, trends, county map"]
|
||||
P2["Reliability Explorer<br/>Make/model comparison"]
|
||||
P3["Next-Test Estimator<br/>Risk prediction"]
|
||||
P4["Data & Methods<br/>Coverage, limitations"]
|
||||
end
|
||||
|
||||
Dashboard --> Pages
|
||||
|
||||
%% Privacy Controls
|
||||
subgraph Privacy["Privacy Controls"]
|
||||
subgraph SPLITS[Private development chronology]
|
||||
direction LR
|
||||
S1["≥100 inspections per cell"]
|
||||
S2["≥100 distinct vehicles"]
|
||||
S3["≥10 outcome records"]
|
||||
S4["No VINs/plates/ZIPs"]
|
||||
CONTEXT[2010-15<br/>history context]
|
||||
TRAIN[2016-22<br/>train]
|
||||
TUNE[2023<br/>tune]
|
||||
CAL[2024<br/>calibrate]
|
||||
HOLDOUT[2025<br/>one-time holdout]
|
||||
SHADOW[2026 partial<br/>shadow context]
|
||||
end
|
||||
MART -.-> SPLITS
|
||||
|
||||
JSON -.- Privacy
|
||||
MART --> LOGISTIC[Regularized logistic + Platt<br/><b>final prototype model</b>]
|
||||
MART --> TREE[Histogram gradient boosting + Platt<br/><b>benchmark only</b>]
|
||||
|
||||
%% Testing
|
||||
subgraph Tests["Test Suite"]
|
||||
LOGISTIC --> PRIVATE[Private model artifacts<br/>manifests, checksums, metrics]
|
||||
TREE --> PRIVATE
|
||||
HOLDOUT -.-> REPORT[Private/report-only<br/>one-time comparison]
|
||||
PRIVATE -.-> REPORT
|
||||
|
||||
MART --> EXPORT[Aggregate, suppress,<br/>round, validate]
|
||||
PRIVATE -.->|pre-2025 diagnostics only| EXPORT
|
||||
|
||||
subgraph PUBLIC[dashboard/public/data - public boundary]
|
||||
direction TB
|
||||
MANIFEST[data_manifest.json]
|
||||
OVERVIEW[overview_period_county.json]
|
||||
COHORTS[cohort_scorecard.json]
|
||||
AGE[age_risk_curve.json]
|
||||
COVERAGE[coverage_quality.json]
|
||||
FILTERS[filter_catalog.json]
|
||||
DIAGNOSTICS[model_diagnostics.json]
|
||||
HASHES[sha256_manifest.json]
|
||||
end
|
||||
EXPORT --> PUBLIC
|
||||
|
||||
PUBLIC --> CONTRACT[Fail-closed browser validation]
|
||||
CONTRACT --> SITE[Static dashboard<br/>semantic HTML + CSS<br/>vanilla ES modules + inline SVG]
|
||||
|
||||
subgraph VIEWS[Presentation-ready views]
|
||||
direction TB
|
||||
V1[Overview]
|
||||
V2[Sample cohorts]
|
||||
V3[Model & benchmark]
|
||||
V4[Data & methods]
|
||||
end
|
||||
SITE --> VIEWS
|
||||
|
||||
subgraph GUARDRAILS[Publication guardrails]
|
||||
direction TB
|
||||
G1[Private 10,000-vehicle sample<br/>not population estimates]
|
||||
G2[Minimum episode, vehicle,<br/>and binary-class support]
|
||||
G3[No identifiers, raw/operational rows,<br/>credentials, or row-level predictions]
|
||||
G4[No county population rankings<br/>or vehicle-level prediction service]
|
||||
end
|
||||
EXPORT -.-> GUARDRAILS
|
||||
|
||||
SITE --> BOLT[Separate dashboard-only<br/>Bolt project]
|
||||
|
||||
subgraph TESTS[Verification]
|
||||
direction LR
|
||||
PT["Python unittest<br/>(feature mart, models,<br/>export, baselines)"]
|
||||
CT["Node.js contract tests<br/>(dashboard JSON schemas)"]
|
||||
PY[Python pipeline/model/export tests]
|
||||
JS[Node dashboard contract tests]
|
||||
end
|
||||
MART -.-> PY
|
||||
PUBLIC -.-> JS
|
||||
|
||||
MART -.- PT
|
||||
JSON -.- CT
|
||||
classDef private fill:#fff3e0,stroke:#b45309,color:#3b2415
|
||||
classDef model fill:#e0f2fe,stroke:#0369a1,color:#0c4a6e
|
||||
classDef public fill:#ecfdf5,stroke:#047857,color:#064e3b
|
||||
classDef warning fill:#fef2f2,stroke:#b91c1c,color:#7f1d1d
|
||||
|
||||
%% Styling
|
||||
classDef source fill:#e1f5fe,stroke:#0288d1
|
||||
classDef process fill:#fff3e0,stroke:#f57c00
|
||||
classDef storage fill:#e8f5e9,stroke:#388e3c
|
||||
classDef dashboard fill:#fce4ec,stroke:#c62828
|
||||
classDef test fill:#f3e5f5,stroke:#7b1fa2
|
||||
|
||||
class PG,DMV,INSP source
|
||||
class CSV,MART,ARTIFACTS,DUCKDB storage
|
||||
class BL,TREE process
|
||||
class SERVER,Dashboard,Pages,JSON dashboard
|
||||
class Tests test
|
||||
class SOURCE,SAMPLE,STAGE,EPISODES,MART,PRIVATE,REPORT private
|
||||
class LOGISTIC,TREE model
|
||||
class PUBLIC,CONTRACT,SITE,VIEWS,BOLT public
|
||||
class GUARDRAILS warning
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
# Dashboard-only Bolt deployment
|
||||
|
||||
Last reviewed: 2026-07-21
|
||||
|
||||
## Non-negotiable publication boundary
|
||||
|
||||
Publish a **separate Bolt project or GitHub repository containing only the
|
||||
contents of `dashboard/`**. In that project, `index.html` must be at the project
|
||||
root.
|
||||
|
||||
Do not import the full Utah Vehicle Health source repository into Bolt. Do not
|
||||
depend on Bolt supporting a `dashboard` working-root setting. The source
|
||||
repository contains private-pipeline structure that a static host does not need,
|
||||
even when local secrets and data are Git-ignored.
|
||||
|
||||
The deployment project must never contain `.env` files, credentials, private
|
||||
data or marts, database files, model artifacts, SQL, pipeline scripts, repository
|
||||
history from outside `dashboard/`, raw JSON, operational records, or row-level
|
||||
predictions.
|
||||
|
||||
## 1. Verify the source dashboard locally
|
||||
|
||||
From the Utah Vehicle Health repository root:
|
||||
|
||||
```bash
|
||||
npm --prefix dashboard test
|
||||
node dashboard/server.mjs
|
||||
```
|
||||
|
||||
Open `http://127.0.0.1:4173` and verify:
|
||||
|
||||
- the development-preview banner says the values come from the private
|
||||
10,000-vehicle sample and are not population estimates;
|
||||
- the routes are Overview, Sample cohorts, Model & benchmark, and Data & methods;
|
||||
- the final model is calibrated logistic regression and the boosted tree is
|
||||
benchmark-only;
|
||||
- model cards show 2024 calibration-fit diagnostics, not the opened 2025
|
||||
holdout comparison;
|
||||
- no vehicle-level prediction service or input form exists;
|
||||
- county displays communicate feed coverage rather than population rankings;
|
||||
and
|
||||
- a missing or invalid aggregate bundle produces an unavailable state, not
|
||||
fallback estimates.
|
||||
|
||||
Stop the local server with `Ctrl-C` after verification.
|
||||
|
||||
## 2. Create a fresh dashboard-only artifact
|
||||
|
||||
Use a new directory and, preferably, a new deployment repository. Copy the
|
||||
**contents** of `dashboard/`, not the directory's parent and not the main
|
||||
repository's `.git` history.
|
||||
|
||||
The reviewed project tree is:
|
||||
|
||||
```text
|
||||
index.html
|
||||
styles.css
|
||||
package.json
|
||||
server.mjs
|
||||
README.md
|
||||
js/
|
||||
app.js
|
||||
charts.js
|
||||
data.js
|
||||
public/data/
|
||||
age_risk_curve.json
|
||||
cohort_scorecard.json
|
||||
coverage_quality.json
|
||||
data_manifest.json
|
||||
filter_catalog.json
|
||||
model_diagnostics.json
|
||||
overview_period_county.json
|
||||
sha256_manifest.json
|
||||
tests/
|
||||
contract.test.mjs
|
||||
```
|
||||
|
||||
Do not add any other repository directory. In the fresh artifact root, run:
|
||||
|
||||
```bash
|
||||
npm test
|
||||
HOST=0.0.0.0 npm start
|
||||
```
|
||||
|
||||
No dependency install or build step is required. The `PORT` environment
|
||||
variable is honored automatically by `server.mjs` when a host supplies it.
|
||||
|
||||
Before creating the deployment repository, inspect its entire file tree. Apart
|
||||
from `package.json` project metadata, the only JSON allowed is the reviewed
|
||||
aggregate bundle under `public/data/`. Confirm that `data_manifest.json` still
|
||||
declares:
|
||||
|
||||
```text
|
||||
development_preview: true
|
||||
population_estimate_allowed: false
|
||||
locked_test_metrics_published: false
|
||||
```
|
||||
|
||||
Do not hand-edit an aggregate file to make validation pass. Regenerate it
|
||||
through the private local exporter and repeat the review.
|
||||
|
||||
## 3. Import only the dashboard repository into Bolt
|
||||
|
||||
The recommended handoff is a fresh GitHub repository containing the reviewed
|
||||
tree above. It may be private if the connected Bolt account has access.
|
||||
|
||||
1. On the Bolt homepage, choose the GitHub import control.
|
||||
2. Select the separate dashboard-only repository or use **Import from URL**.
|
||||
3. Confirm `index.html` appears at the imported project root.
|
||||
4. Inspect the Bolt file tree before previewing. If SQL, scripts, artifacts,
|
||||
private data, `.env`, or source-repository files appear, stop and delete that
|
||||
Bolt project; create a clean dashboard-only project instead.
|
||||
5. Use `HOST=0.0.0.0 npm start` if Bolt requests a preview command. There is no
|
||||
build command.
|
||||
|
||||
Bolt's current GitHub import workflow is documented in
|
||||
[GitHub for version control](https://support.bolt.new/integrations/git).
|
||||
|
||||
If a separate GitHub repository is not used, create a fresh Bolt/StackBlitz
|
||||
project and upload only the reviewed dashboard files so the same tree appears at
|
||||
the project root. Never upload the full source repository and then try to hide
|
||||
or ignore its private-pipeline directories.
|
||||
|
||||
## 4. Review in Bolt before publishing
|
||||
|
||||
Use the preview URL and repeat the local visual checks. Also verify directly:
|
||||
|
||||
- `/public/data/data_manifest.json` resolves;
|
||||
- an unknown file path returns `404`;
|
||||
- `/.env`, `/data/`, `/artifacts/`, `/models/`, `/sql/`, and `/scripts/` return
|
||||
`404`;
|
||||
- the browser makes requests only for the static HTML, CSS, JavaScript, and
|
||||
approved files under `/public/data/`; and
|
||||
- no analytics, database, authentication, or third-party data service was added
|
||||
by an automated Bolt edit.
|
||||
|
||||
Preview privately first. If Bolt proposes code changes, review the full diff and
|
||||
rerun `npm test`; do not accept changes that add a database, environment secret,
|
||||
tracking script, model API, prediction form, or remote data source.
|
||||
|
||||
## 5. Publish with Bolt Hosting
|
||||
|
||||
Bolt Hosting is the intended host for this prototype:
|
||||
|
||||
1. Open the dashboard-only project.
|
||||
2. Select **Publish** in the upper-right corner.
|
||||
3. Choose private visibility for presentation review when available.
|
||||
4. Select **Publish** and wait for the generated `bolt.host` URL.
|
||||
5. Run the post-publication checks below before changing visibility to public.
|
||||
|
||||
Bolt documents this flow in
|
||||
[Publish your project to a live website](https://support.bolt.new/cloud/hosting/publish).
|
||||
Bolt's Netlify integration is optional and is not required for this dashboard;
|
||||
see [Netlify integration](https://support.bolt.new/integrations/netlify) only if
|
||||
hosting requirements explicitly change.
|
||||
|
||||
## 6. Post-publication checks
|
||||
|
||||
At the exact published URL:
|
||||
|
||||
1. Hard-refresh and confirm the development-preview banner remains visible.
|
||||
2. Visit all four routes and verify labels, charts, and keyboard navigation.
|
||||
3. Confirm the Model & benchmark view identifies logistic as final, tree as
|
||||
benchmark, and its values as 2024 calibration-fit checks.
|
||||
4. Confirm no 2025 holdout metric appears in the site or public JSON.
|
||||
5. Confirm there is no vehicle-level lookup, input form, or row-level output.
|
||||
6. Confirm coverage language does not imply statewide representation or county
|
||||
ranking.
|
||||
7. Request the denied paths from step 4 again and require `404`.
|
||||
8. Save the published URL, release ID, publication date, and reviewer decision;
|
||||
do not save private paths, row values, or credentials in the release note.
|
||||
|
||||
If any check fails, use Bolt's Publish menu to unpublish or keep the site private
|
||||
until a corrected dashboard-only artifact passes local and hosted review.
|
||||
|
||||
## Updating a published prototype
|
||||
|
||||
Changes in a Bolt project are not automatically live. For each update:
|
||||
|
||||
1. regenerate/review aggregates locally;
|
||||
2. run the dashboard contract test;
|
||||
3. update the separate dashboard-only repository/project;
|
||||
4. inspect Bolt's diff and preview;
|
||||
5. use **Publish > Update**; and
|
||||
6. repeat every post-publication check.
|
||||
|
||||
Never synchronize private pipeline directories into the deployment repository.
|
||||
+101
-72
@@ -1,97 +1,126 @@
|
||||
# Bolt dashboard specification
|
||||
# Static dashboard specification
|
||||
|
||||
## Product language
|
||||
## Release status and language
|
||||
|
||||
Use **Utah Vehicle Health** as the brand, but call the modeled quantity
|
||||
**next-episode non-pass risk**. The estimator is a cohort estimate, not a
|
||||
diagnosis, certification, or guarantee.
|
||||
The checked-in site is a **development preview** built from suppressed
|
||||
aggregates derived from the private, page-sampled 10,000-vehicle development
|
||||
cohort. Every value is a sample result, **not a population estimate**. The site
|
||||
must not present statewide prevalence, county rankings, causal comparisons, or
|
||||
individual predictions.
|
||||
|
||||
## Four-page MVP
|
||||
Use **Utah Vehicle Health** as the project name and **next-episode first-attempt
|
||||
non-pass** for the modeled outcome. Non-pass combines fail, reject, and abort.
|
||||
The dashboard is not a diagnosis, certification, safety assessment,
|
||||
roadworthiness assessment, or guarantee.
|
||||
|
||||
The sections below describe the full product target. The checked-in
|
||||
development preview intentionally implements a narrower safe subset: binary
|
||||
pass/non-pass aggregates, supported make/model scorecards, age bands, coverage
|
||||
quality, and pre-2025 model diagnostics. Four-class charts, uncertainty
|
||||
intervals, adjusted scorecards, and the prediction lookup remain disabled until
|
||||
their own reviewed aggregate assets exist.
|
||||
## Implemented four-view prototype
|
||||
|
||||
### Overview
|
||||
|
||||
- Eligible inspections, pass rate, non-pass rate, and covered-period KPIs
|
||||
- Quarterly pass/fail/reject/abort trend with blanks shown separately
|
||||
- Covered-county map; unavailable counties remain gray
|
||||
- Non-pass risk versus vehicle age with intervals
|
||||
- Clear notices for partial periods and limited feed coverage
|
||||
- Visibly persistent development-preview warning
|
||||
- Rounded published support and sample pass/non-pass rates
|
||||
- Sample first-attempt non-pass trend for approved, pre-2025 periods
|
||||
- County **feed-coverage** map, with unavailable counties shown as unavailable
|
||||
rather than zero
|
||||
- Sample non-pass pattern by coarsened vehicle-age band
|
||||
- Plain-language target and limitations callout
|
||||
|
||||
### Reliability explorer
|
||||
County values provide sample/feed context only. They must not be sorted,
|
||||
headlined, or described as population county performance.
|
||||
|
||||
- Search supported canonical make/model cohorts
|
||||
- Compare up to three cohorts across vehicle-age bands
|
||||
- Observed versus model-adjusted risk toggle
|
||||
- Outcome-mix bar and uncertainty-aware ranked dot plot
|
||||
- County, make/model, age band, fuel, program, and period filters
|
||||
- Support size and interval displayed for every estimate
|
||||
### Sample cohorts
|
||||
|
||||
### Next-test risk estimator
|
||||
- Search and compare only supported, suppression-cleared make/model cohorts
|
||||
- Show observed sample non-pass rates and rounded support
|
||||
- Default sorting by support rather than risk
|
||||
- Disable filters or adjusted views that the aggregate bundle cannot support
|
||||
- Explain that omitted cohorts may be suppressed or unavailable
|
||||
|
||||
Inputs are coarsened, non-identifying attributes: county, supported make/model,
|
||||
vehicle-age band, fuel, prior episode outcome, time-since-prior band, season,
|
||||
and approved program category.
|
||||
These are descriptive development-sample cohorts, not reliability ratings,
|
||||
causal make/model effects, or recommendations.
|
||||
|
||||
Output a calibrated non-pass probability, uncertainty interval, relevant
|
||||
baseline, and aggregate factor contributions. Never request VIN, plate, exact
|
||||
address, station, free text, or current-test diagnostics.
|
||||
### Model & benchmark
|
||||
|
||||
### Data and methods
|
||||
- Identify calibrated logistic regression as the sole final prototype model.
|
||||
- Identify calibrated histogram gradient boosting as benchmark-only.
|
||||
- Show only pre-2025, 2024 calibration-fit diagnostics from the private sample.
|
||||
- Explain that those same-partition values are calibration checks, not
|
||||
independent final-performance or population estimates.
|
||||
- Keep the opened 2025 holdout comparison in the private report/presentation,
|
||||
not in the browser bundle.
|
||||
- State plainly that the finished prototype has no vehicle-level prediction
|
||||
service, personal input form, or row-level output.
|
||||
|
||||
- Coverage timeline and source/year missingness heatmap
|
||||
- Episode and target definitions
|
||||
- `slc`/`slco` source-era explanation
|
||||
- Temporal split, PR-AUC, Brier score, and calibration plot
|
||||
- Subgroup/source-era performance
|
||||
- Leakage controls, DMV gaps, partial periods, and limitations
|
||||
### Data & methods
|
||||
|
||||
- First-attempt returning-vehicle episode and binary-target definitions
|
||||
- Point-in-time feature and leakage controls
|
||||
- Coverage timeline by source era
|
||||
- Chronological development-sample split and one-time 2025-gate disclosure
|
||||
- Evidence-governance summary for the logistic final model and tree benchmark
|
||||
- Known sampling, coverage, label, validation, and interpretation limits
|
||||
- Private-to-public publication flow
|
||||
|
||||
The boosted tree appears only in benchmark context, never as a second final
|
||||
model. The dashboard never requests a VIN, plate, ZIP, exact address, station,
|
||||
technician, free text, or current-test diagnostic.
|
||||
|
||||
## Public data contract
|
||||
|
||||
| Dataset | Safe grain |
|
||||
| Asset | Allowed content |
|
||||
| --- | --- |
|
||||
| `data_manifest` | Data cutoff, deterministic release ID, model versions, definitions and exclusions |
|
||||
| `overview_period_county` | Quarter/year × public county with rounded support and outcome rates |
|
||||
| `cohort_scorecard` | Approved make/model × age band, optionally coarsened county/fuel |
|
||||
| `age_risk_curve` | Approved cohort × age point/band with risk, interval and support |
|
||||
| `prediction_lookup` | Only supported coarsened input combinations and calibrated outputs |
|
||||
| `filter_catalog` | Publishable categories and valid combinations |
|
||||
| `model_diagnostics` | Approved partition-level metrics; locked metrics require a separate release gate |
|
||||
| `coverage_quality` | Source era × year volume, blank rate, linkage and availability |
|
||||
| `data_manifest.json` | Release ID, model versions, scope flags, definitions, and suppression settings |
|
||||
| `overview_period_county.json` | Suppressed quarter x covered-feed county sample aggregates |
|
||||
| `cohort_scorecard.json` | Suppressed make/model sample aggregates with rounded support |
|
||||
| `age_risk_curve.json` | Suppressed sample rates by coarsened vehicle-age band |
|
||||
| `filter_catalog.json` | Publishable categories and supported periods |
|
||||
| `model_diagnostics.json` | Train/tune/calibration development metrics only; no 2025 metrics |
|
||||
| `coverage_quality.json` | Source-era availability by year |
|
||||
| `sha256_manifest.json` | Checksums for the approved bundle |
|
||||
|
||||
Use purpose-built outputs rather than a single high-dimensional browser cube.
|
||||
Bolt receives only these sanitized, versioned assets—never countydata
|
||||
credentials or private analytical rows.
|
||||
Every JSON asset must declare `development_preview=true` and
|
||||
`population_estimate_allowed=false`. The public manifest must declare
|
||||
`locked_test_metrics_published=false`. The browser validates the complete bundle
|
||||
and shows no estimates if any required asset fails its contract.
|
||||
|
||||
## Publication controls
|
||||
The browser bundle must never contain raw analytical rows, prediction lookup
|
||||
rows, hidden unsuppressed chart layers, or values recoverable only through the
|
||||
developer console.
|
||||
|
||||
- Suppress cells below 100 eligible inspections.
|
||||
- Also require at least 100 distinct private vehicle tokens in every published
|
||||
cell; tokens and distinct counts never enter the public asset.
|
||||
- Suppress when an outcome or its complement has fewer than 10 records.
|
||||
- Require at least 10 distinct vehicles contributing each binary class.
|
||||
- Apply complementary suppression so totals cannot reconstruct hidden cells.
|
||||
- Combine rare categories, coarsen model years, and round displayed counts.
|
||||
- Recheck thresholds after every filter combination.
|
||||
- Do not include suppressed rows in browser bundles, API responses, downloads,
|
||||
analytics logs, or hidden chart layers.
|
||||
- Downloads contain only the sanitized summary currently displayed.
|
||||
## Suppression and privacy controls
|
||||
|
||||
## Visual direction
|
||||
- Require at least 100 eligible episodes and 100 distinct private vehicles per
|
||||
published cell.
|
||||
- Require at least 10 observations and 10 distinct vehicles in both binary
|
||||
outcome classes.
|
||||
- Apply complementary suppression when a visible total could reveal a hidden
|
||||
cell.
|
||||
- Combine rare categories, coarsen age/year fields, and round support.
|
||||
- Recheck thresholds for the exact grain of every released asset.
|
||||
- Omit suppressed rows entirely; do not send them to the browser.
|
||||
- Publish no VIN, plate, ZIP, exact address, station, technician identifier,
|
||||
private token, raw JSON, credential, operational record, free text, or
|
||||
row-level prediction.
|
||||
- Make no browser or hosted-service connection to `countydata`.
|
||||
|
||||
Use a restrained Utah/desert palette: teal pass, red fail, amber reject, purple
|
||||
abort, and gray missing. Use probability bars, calibrated dot plots, confidence
|
||||
bands, and cohort comparisons instead of gauges. Do not rely on color alone.
|
||||
## Accessibility and visual behavior
|
||||
|
||||
## Stretch pages
|
||||
- Use semantic headings, landmarks, labels, status messages, and keyboard-
|
||||
reachable navigation.
|
||||
- Provide text alternatives or accessible labels for charts.
|
||||
- Do not rely on color alone.
|
||||
- Use a restrained Utah/desert palette and direct probability/rate encodings;
|
||||
avoid gauges and decorative risk scores.
|
||||
- Preserve the warning banner and fail-closed unavailable state on desktop and
|
||||
mobile layouts.
|
||||
|
||||
- Failure-to-pass journeys with funnels, attempts-to-pass and survival curves
|
||||
- Four-class outcome probabilities
|
||||
- Source-scoped OBD early-warning analysis
|
||||
- Automated aggregate refresh with a dedicated read-only role
|
||||
- Model-drift monitoring
|
||||
## Deployment boundary
|
||||
|
||||
Bolt receives a separate dashboard-only project whose root is the contents of
|
||||
`dashboard/`. Do not import or upload the full private-pipeline repository and
|
||||
do not rely on a configurable subdirectory working root. The deployable project
|
||||
contains no `.env`, private `data/`, model artifacts, SQL, pipeline scripts, or
|
||||
repository history outside the dashboard directory.
|
||||
|
||||
See [bolt_deployment.md](bolt_deployment.md) for the reviewed handoff and
|
||||
post-publication checks.
|
||||
|
||||
+10
-3
@@ -1,8 +1,15 @@
|
||||
# Countydata inventory
|
||||
|
||||
Inventory date: 2026-07-15; live schema revalidated 2026-07-21. All inspection
|
||||
was performed with read-only transactions, metadata queries, aggregate queries,
|
||||
and bounded JSON field-path sampling; no identifier values were exported.
|
||||
Inventory date: 2026-07-15; live schema revalidated 2026-07-21. All inventory
|
||||
work used read-only transactions, metadata queries, aggregate queries, and
|
||||
bounded JSON field-path sampling; no identifier values were exported.
|
||||
|
||||
> **Source-discovery context only:** Database-wide metadata and aggregate counts
|
||||
> below describe the accessible source, not Utah Vehicle Health research
|
||||
> results. The finished prototype uses a private, page-sampled 10,000-vehicle
|
||||
> development cohort; all model/dashboard findings are sample-based and not
|
||||
> population estimates. See the authoritative [project charter](project_charter.md)
|
||||
> and [final report](final_report.md).
|
||||
|
||||
## Live access and full-schema boundary
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# Utah Vehicle Health dashboard demo script
|
||||
|
||||
Target length: **2 minutes 45 seconds**, embedded in the
|
||||
[10-minute presentation](presentation_outline.md).
|
||||
|
||||
## Before the audience arrives
|
||||
|
||||
1. From the repository root, run `node dashboard/server.mjs`.
|
||||
2. Open `http://127.0.0.1:4173/#overview`.
|
||||
3. Confirm the header says the sample aggregates validated.
|
||||
4. Confirm the private-sample development-preview banner is visible.
|
||||
5. Visit all four routes: Overview, Sample cohorts, Model & benchmark, and Data
|
||||
& methods.
|
||||
6. Reset Sample cohorts and leave its sort on largest support.
|
||||
|
||||
Do not open developer tools, private files, model artifacts, database clients,
|
||||
or environment variables during the presentation.
|
||||
|
||||
## Live talk track
|
||||
|
||||
### 0:00-0:25 — Establish the boundary
|
||||
|
||||
**Action:** Start on **Overview**. Point to the development-preview banner before
|
||||
pointing to a chart.
|
||||
|
||||
**Say:**
|
||||
|
||||
> The most important element is this banner. Every value in the dashboard comes
|
||||
> from the private 10,000-vehicle development sample. These are suppression-
|
||||
> reviewed sample aggregates, not population estimates, and they cannot support
|
||||
> individual decisions or county rankings.
|
||||
|
||||
### 0:25-1:00 — Explain the overview
|
||||
|
||||
**Action:** Point to published support and the observed first-attempt non-pass
|
||||
trend, then the coverage map and vehicle-age chart. Do not rank counties by
|
||||
outcome.
|
||||
|
||||
**Say:**
|
||||
|
||||
> The overview reports rounded support and sample first-attempt outcomes for
|
||||
> returning vehicles. Non-pass combines fail, reject, and abort. The trend is
|
||||
> descriptive of sampled records only. This county graphic communicates feed
|
||||
> availability: gray means unavailable, not zero and not better. The age pattern
|
||||
> is also an observed sample association, not a causal claim or diagnosis.
|
||||
|
||||
### 1:00-1:35 — Use Sample cohorts safely
|
||||
|
||||
**Action:** Open **Sample cohorts**. Search for `Toyota` or another currently
|
||||
supported make. Leave sorting on largest support. Point to rounded support and
|
||||
the “observed sample non-pass” label.
|
||||
|
||||
**Say:**
|
||||
|
||||
> This view contains only prior make-and-model cohorts that cleared the
|
||||
> publication thresholds. I can search supported cohorts, but these are observed
|
||||
> development-sample associations—not reliability grades, rankings, or
|
||||
> recommendations. Unsupported slices are not inferred in the browser, and
|
||||
> suppressed rows are absent rather than hidden.
|
||||
|
||||
**Action:** Reset the search before leaving the view.
|
||||
|
||||
### 1:35-2:15 — Separate the final model from its benchmark
|
||||
|
||||
**Action:** Open **Model & benchmark**. Point first to the role badges, then the
|
||||
2024 calibration-fit scope strip, then the “No vehicle-level prediction service”
|
||||
boundary.
|
||||
|
||||
**Say:**
|
||||
|
||||
> The governance decision is explicit: calibrated logistic regression is the
|
||||
> final prototype model, while the boosted tree is benchmark-only. The numbers
|
||||
> shown here are pre-2025 checks on the same 2024 sample partition used to fit
|
||||
> calibration. They are not independent final-performance or population
|
||||
> estimates, so we do not select between the cards from these values. The
|
||||
> one-time 2025 comparison shown earlier favored logistic and stays in the report
|
||||
> and presentation, not the browser bundle. This finished dashboard contains no
|
||||
> vehicle lookup, personal inputs, or row-level prediction output.
|
||||
|
||||
### 2:15-2:40 — Close on method and privacy
|
||||
|
||||
**Action:** Open **Data & methods**. Briefly point to Prediction unit, the
|
||||
chronological timeline, evidence status, and the privacy flow.
|
||||
|
||||
**Say:**
|
||||
|
||||
> The method page makes the contract visible: the target is the first attempt of
|
||||
> a returning vehicle's next episode, and every feature ends before that episode.
|
||||
> Training, tuning, and calibration are chronological. It also records that 2025
|
||||
> was opened once after choices were frozen. Finally, the public path ends in
|
||||
> suppressed summaries. No private rows or database connection reach this site.
|
||||
|
||||
### 2:40-2:45 — Transition
|
||||
|
||||
**Say:**
|
||||
|
||||
> That is the prototype: useful sample evidence, with model and privacy
|
||||
> boundaries kept visible.
|
||||
|
||||
Return to the slide deck's privacy architecture.
|
||||
|
||||
## If something goes wrong
|
||||
|
||||
If the dashboard shows **data unavailable**, do not bypass validation or edit
|
||||
JSON. Say:
|
||||
|
||||
> The site rejected an aggregate-contract mismatch and is failing closed, so it
|
||||
> shows no estimates. That behavior is part of the privacy and integrity design.
|
||||
|
||||
Then continue with a backup screenshot or slides. If a cohort search returns no
|
||||
rows, explain that unsupported or suppressed cohorts are intentionally absent
|
||||
and reset the filter. Never substitute remembered values or improvise a vehicle-
|
||||
level example.
|
||||
+88
-93
@@ -1,67 +1,66 @@
|
||||
# Development run results
|
||||
|
||||
Run date: 2026-07-15
|
||||
Run date: 2026-07-15; presentation wording reviewed 2026-07-21
|
||||
|
||||
These results validate the engineering and modeling pipeline. They are **not
|
||||
population estimates**: the development extract samples inspection-table pages
|
||||
and then retrieves complete histories for up to 10,000 vehicles, which
|
||||
over-represents vehicles with more inspection records.
|
||||
> **Scope of every value below:** the private, page-sampled 10,000-vehicle
|
||||
> development cohort. These are sample results, not population estimates. Page
|
||||
> sampling over-represents vehicles with more inspection records. Nothing in
|
||||
> this document supports statewide/county prevalence or county rankings.
|
||||
|
||||
## Label audit
|
||||
|
||||
The raw Utah County feed leaves `overall_result` blank for 3,747,862 records.
|
||||
An aggregate-only server audit found 3,469,102 Utah `obd`/`OBD` records with a
|
||||
controlled pass, fail, reject, or abort value in `obd_result`. Across non-Utah
|
||||
feeds where both fields were recognized, the two fields agreed on binary pass
|
||||
versus non-pass 99.30% of the time, but their fail-versus-reject categories were
|
||||
not interchangeable.
|
||||
The binary target is pass versus non-pass, where non-pass combines recognized
|
||||
fail, reject, and abort outcomes. Blanks and unrecognized values are unlabeled,
|
||||
not passes.
|
||||
|
||||
Label contract v3 therefore permits the Utah OBD value only as a provenance-
|
||||
tagged binary proxy. TSI, `other/C`, `B`, blank, and unknown values stay
|
||||
unlabeled. Four-class analysis must use `overall_result` exclusively. This also
|
||||
matches the official program distinction between a readiness rejection and a
|
||||
failed inspection; see the [Utah inspection requirements](https://dmv.utah.gov/register/inspections/)
|
||||
and [program definition of rejection](https://www.utah.gov/pmn/files/1155003.pdf).
|
||||
|
||||
In the development extract, the rule restored 11,334 labels and left 1,281
|
||||
events unlabeled. The label-source field remains private audit metadata and is
|
||||
not a model feature.
|
||||
For the older Utah County feed, the documented source/program/test rule permits
|
||||
a controlled OBD result to fill a blank overall result as a binary proxy. It
|
||||
does not support a four-class interpretation. In the private 10,000-vehicle
|
||||
sample, the rule restored 11,334 labels and left 1,281 events unlabeled. These
|
||||
sample counts are not population estimates. Label provenance stays in private
|
||||
audit metadata and is not a predictor.
|
||||
|
||||
## Pipeline reconciliation
|
||||
|
||||
| Stage | Rows |
|
||||
This table reports private 10,000-vehicle development-sample pipeline counts,
|
||||
not population totals.
|
||||
|
||||
| Sample pipeline stage | Rows |
|
||||
| --- | ---: |
|
||||
| Source events written after VIN validation | 83,552 |
|
||||
| Source events written after identifier validation | 83,552 |
|
||||
| Clean events after duplicate/conflict handling | 80,190 |
|
||||
| Inspection episodes | 69,588 |
|
||||
| Eligible returning targets | 44,659 |
|
||||
| Eligible returning-vehicle targets | 44,659 |
|
||||
|
||||
The sample contains 9,996 retained vehicle tokens. The extraction manifest,
|
||||
compressed-file SHA-256, mart manifest, Parquet SHA-256, and row counts all
|
||||
reconcile. Every mart invariant reports zero violations. Raw VIN and raw OBD
|
||||
result are absent from the staging output schema, feature mart, and model
|
||||
features.
|
||||
The sample retained 9,996 private vehicle tokens. Extraction and mart manifests,
|
||||
compressed-file and Parquet checksums, and stage counts reconcile. Every mart
|
||||
invariant reports zero violations. Raw identifiers and raw OBD values are absent
|
||||
from staging output, feature mart predictors, and model features.
|
||||
|
||||
| Temporal partition | Eligible targets | Never-fit audit targets |
|
||||
The following partition counts are also from the private 10,000-vehicle
|
||||
development sample and are not population estimates.
|
||||
|
||||
| Sample temporal partition | Eligible targets | Never-fit audit targets |
|
||||
| --- | ---: | ---: |
|
||||
| Train, 2016–2022 | 31,745 | 3,129 |
|
||||
| Train, 2016-2022 | 31,745 | 3,129 |
|
||||
| Tune, 2023 | 4,972 | 526 |
|
||||
| Calibrate, 2024 | 2,633 | 277 |
|
||||
| Locked test, 2025 | 4,451 | 480 |
|
||||
| One-time holdout, 2025 | 4,451 | 480 |
|
||||
| Shadow, 2026 partial | 858 | 106 |
|
||||
|
||||
Audit-bucket vehicles are excluded from fitting, tuning, and calibration. The
|
||||
normal trainers do not calculate 2025 outcomes or metrics without the explicit
|
||||
Audit-bucket vehicles are excluded from fitting, tuning, and calibration. Normal
|
||||
training does not calculate 2025 outcomes or metrics without the explicit
|
||||
locked-evaluation flag.
|
||||
|
||||
## Baseline results before the locked test
|
||||
## Pre-2025 development diagnostics
|
||||
|
||||
The table below uses only non-audit vehicles. Average precision is the project's
|
||||
PR-AUC summary. The calibrated logistic row is shown only on the partition used
|
||||
to fit the calibrator and is therefore a calibration diagnostic, not an
|
||||
independent final estimate.
|
||||
The table uses non-audit rows from the private 10,000-vehicle development
|
||||
sample. Average precision is the project's PR-AUC summary. The calibrated 2024
|
||||
row is evaluated on the same sample partition used to fit Platt scaling, so it
|
||||
is a calibration diagnostic rather than independent final performance. None of
|
||||
these values is a population estimate.
|
||||
|
||||
| Partition | Model | PR-AUC | Brier | ROC-AUC | Precision at top 10% |
|
||||
| Sample partition | Model | PR-AUC | Brier | ROC-AUC | Precision at top 10% |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| 2023 tune | Training prevalence | 0.125 | 0.1092 | 0.500 | 0.125 |
|
||||
| 2023 tune | Previous episode, literal | 0.166 | 0.1723 | 0.582 | 0.285 |
|
||||
@@ -71,66 +70,62 @@ independent final estimate.
|
||||
| 2024 calibrate | Logistic | 0.257 | 0.0936 | 0.690 | 0.308 |
|
||||
| 2024 calibrate | Logistic + Platt | **0.257** | **0.0936** | **0.690** | **0.308** |
|
||||
|
||||
The selected logistic regularization was `C=0.03`. It converged in 2,390 of the
|
||||
5,000 allowed iterations; Platt calibration converged in five iterations. All
|
||||
five candidates and all stored probabilities passed explicit convergence and
|
||||
finite-value checks.
|
||||
The selected logistic regularization is `C=0.03`. It converged in 2,390 of the
|
||||
5,000 allowed iterations; Platt calibration converged in five iterations. These
|
||||
are development-run diagnostics. All candidates and stored probabilities passed
|
||||
explicit convergence and finite-value checks.
|
||||
|
||||
The literal previous-outcome baseline is useful as a ranking sanity check but
|
||||
produces overconfident zero/one probabilities, explaining its poor Brier and log
|
||||
loss. The transparent logistic model is the current development leader: on the
|
||||
2023 tuning partition it more than doubles prevalence PR-AUC and raises top-10%
|
||||
precision from 12.5% to 31.5%.
|
||||
On the 2023 sample partition, logistic regression more than doubled prevalence
|
||||
PR-AUC and raised top-10% precision from 12.5% to 31.5%. This describes the
|
||||
private development sample only.
|
||||
|
||||
## One-time development holdout
|
||||
## One-time development-sample holdout
|
||||
|
||||
The nonlinear live-verification command invoked the explicit 2025 gate after
|
||||
its feature contract and four-candidate grid had already been fixed and its
|
||||
candidate had been selected only on 2023. No test-driven model change was made.
|
||||
At that point the development specification was frozen and the already-fixed
|
||||
logistic model was evaluated once for a direct comparison. These are
|
||||
development-sample holdout results, not final population claims.
|
||||
The nonlinear verification command invoked the explicit 2025 gate only after
|
||||
the feature contract and benchmark grid had been frozen and candidate choices
|
||||
had been made from pre-2025 sample data. No model was changed in response. These
|
||||
are one-time development-sample holdout results, not pristine future-test or
|
||||
population estimates.
|
||||
|
||||
| 2025 non-audit model | PR-AUC | Brier | ROC-AUC | Precision at top 10% |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Training prevalence | 0.123 | 0.1076 | 0.500 | 0.123 |
|
||||
| Previous episode, literal | 0.158 | 0.1731 | 0.576 | 0.268 |
|
||||
| Logistic + Platt | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| Histogram gradient boosting + Platt | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
| 2025 private-sample model | Role | PR-AUC | Brier | ROC-AUC | Precision at top 10% |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| Training prevalence | Baseline | 0.123 | 0.1076 | 0.500 | 0.123 |
|
||||
| Previous episode, literal | Baseline | 0.158 | 0.1731 | 0.576 | 0.268 |
|
||||
| Logistic + Platt | **Final model** | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| Histogram gradient boosting + Platt | Benchmark only | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
|
||||
The holdout contains 3,971 non-audit episodes from 3,734 vehicles. The
|
||||
never-fit audit contains another 480 episodes from 450 vehicles; calibrated
|
||||
logistic PR-AUC is 0.253 and Brier is 0.0859 there. The logistic model beats
|
||||
both simple baselines on the project's headline metrics and remains the chosen
|
||||
development model. The tree did not provide a decisive pre-test improvement
|
||||
that justified its added complexity, and no further 2025-informed tuning is
|
||||
permitted.
|
||||
The sample holdout contains 3,971 non-audit episodes from 3,734 vehicles. Its
|
||||
never-fit audit adds 480 episodes from 450 vehicles; calibrated logistic PR-AUC
|
||||
is 0.253 and Brier score is 0.0859 there. All are private-sample diagnostics,
|
||||
not population estimates or external validation.
|
||||
|
||||
## Remaining gates
|
||||
|
||||
1. Run a complete, contiguous bounded extraction for publishable population
|
||||
aggregates; the page-sampled development cohort cannot support dashboard
|
||||
prevalence or county rankings.
|
||||
2. Treat any later full-data 2025 result as confirmatory rather than a pristine
|
||||
unseen test, because the development sample's holdout has now been opened.
|
||||
3. Run the frozen subgroup/source-era report and clustered uncertainty
|
||||
analysis on the complete extraction.
|
||||
4. Replace the checked-in development-preview bundle with complete-data
|
||||
aggregates only after the population-publication review passes.
|
||||
The calibrated logistic regression remains the final development model. It
|
||||
beats both simple baselines on the headline metrics and beats the boosted-tree
|
||||
benchmark in the one-time comparison. The benchmark did not justify its added
|
||||
complexity. No further 2025-informed tuning is permitted.
|
||||
|
||||
## Sanitized dashboard preview
|
||||
|
||||
The repository now includes a fail-closed exporter and a static four-view
|
||||
dashboard shell. The preview exporter reads only train, tune, and calibration
|
||||
rows dated before 2025. It rejects any model manifest that says the holdout was
|
||||
evaluated, rejects any metric row outside the three approved partitions, and
|
||||
marks every asset `development_preview=true` and
|
||||
`population_estimate_allowed=false`.
|
||||
The static preview uses only suppression-cleared sample aggregates and
|
||||
development diagnostics dated before 2025. It intentionally excludes the
|
||||
one-time holdout metrics. Every asset declares `development_preview=true` and
|
||||
`population_estimate_allowed=false`; the manifest declares
|
||||
`locked_test_metrics_published=false`.
|
||||
|
||||
Published cells must clear minimum episode and distinct-vehicle thresholds,
|
||||
including both binary-class complements. Supports are rounded, direct and
|
||||
pseudonymous identifiers stay private, and a checksum manifest covers the
|
||||
approved JSON bundle. Private model manifests bind the exact metrics files, and
|
||||
the public manifest exposes a deterministic release ID plus model versions for
|
||||
provenance. The site disables the estimator because no privacy-reviewed
|
||||
prediction lookup exists.
|
||||
Published cells must clear minimum episode, distinct-vehicle, and binary-class
|
||||
thresholds. Supports are rounded, suppressed rows are omitted, and a checksum
|
||||
manifest covers the approved JSON bundle. The site fails closed if a contract
|
||||
check fails and contains no vehicle-level prediction service.
|
||||
|
||||
No VIN, plate, ZIP, station, technician identifier, private token, raw JSON,
|
||||
credential, operational record, or row-level prediction is in the public
|
||||
bundle.
|
||||
|
||||
## Boundary of the finished prototype
|
||||
|
||||
The repository completes a presentation-ready development prototype, not a
|
||||
population study or production service. A complete population-frame extraction,
|
||||
fresh external/future validation, individualized prediction lookup, DMV
|
||||
enrichment, multiclass modeling, and operational monitoring are outside this
|
||||
prototype's deliverables. They would require separate approval, analysis, and
|
||||
privacy review; they are not implied by the current results.
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
# Utah Vehicle Health: final development-prototype report
|
||||
|
||||
Prepared July 21, 2026
|
||||
|
||||
> **Result scope:** Every analytical count, rate, chart, and model metric in this
|
||||
> report comes from the private, page-sampled 10,000-vehicle development cohort.
|
||||
> These are sample results, not population estimates. The cohort cannot support
|
||||
> statewide or county prevalence, county rankings, causal conclusions, or
|
||||
> decisions about an individual vehicle.
|
||||
|
||||
## Executive summary
|
||||
|
||||
Utah Vehicle Health asks one focused question:
|
||||
|
||||
> Using only information available before an inspection episode begins, how
|
||||
> well can a calibrated logistic regression estimate whether a returning
|
||||
> vehicle's next episode will have a first-attempt non-pass outcome?
|
||||
|
||||
The finished prototype builds leakage-safe inspection episodes, creates
|
||||
point-in-time history features, evaluates models chronologically, and publishes
|
||||
only suppression-reviewed sample aggregates to a static dashboard. Non-pass
|
||||
combines recognized fail, reject, and abort outcomes; blank or unrecognized
|
||||
results are not treated as passes.
|
||||
|
||||
The final model is regularized logistic regression with Platt calibration. On
|
||||
the private sample's one-time 2025 holdout, it achieved PR-AUC 0.261 and Brier
|
||||
score 0.1011. The calibrated histogram gradient-boosted benchmark achieved
|
||||
PR-AUC 0.238 and Brier score 0.1024 on the same sample holdout. Logistic
|
||||
regression remains the final model because it was both more transparent and
|
||||
better on the declared headline metrics in this development comparison.
|
||||
|
||||
Those values demonstrate that the pipeline and model comparison work on this
|
||||
development sample. They do not establish population performance, production
|
||||
readiness, or external validity.
|
||||
|
||||
## Research scope
|
||||
|
||||
The prediction unit is the first attempt of the next inspection episode for a
|
||||
returning vehicle. Consecutive attempts no more than 30 days apart form one
|
||||
episode. This prevents a fail followed quickly by a retest from becoming two
|
||||
nominally independent prediction targets. A vehicle is eligible only after a
|
||||
prior episode has been observed.
|
||||
|
||||
The binary outcome is:
|
||||
|
||||
- pass; or
|
||||
- non-pass, combining fail, reject, and abort.
|
||||
|
||||
Reject and abort can reflect process or readiness conditions, so this report
|
||||
does not equate non-pass with mechanical failure. Cold-start prediction,
|
||||
four-class modeling, fail-only sensitivity analysis, DMV enrichment,
|
||||
station-level analysis, and individual prediction are outside the completed
|
||||
prototype.
|
||||
|
||||
## Development data
|
||||
|
||||
The private extraction samples inspection-table pages, selects up to 10,000
|
||||
vehicles, and then retrieves their complete available inspection histories. It
|
||||
retained 9,996 private vehicle tokens after validation. The resulting sample
|
||||
contains 83,552 source events, 69,588 episodes, and 44,659 eligible returning-
|
||||
vehicle targets. These figures are pipeline-reconciliation counts for the
|
||||
private sample, not estimates of Utah inspection volume.
|
||||
|
||||
The sample design intentionally makes end-to-end development manageable, but it
|
||||
over-represents vehicles with more inspection records. Participating feeds also
|
||||
change over time and do not cover all 29 Utah counties. The dashboard therefore
|
||||
labels every value as a development-sample aggregate and uses county only to
|
||||
communicate covered-feed context—not to publish population rankings.
|
||||
|
||||
The extraction and mart manifests bind hashes, configurations, and row counts.
|
||||
Private vehicle linkage uses a one-way keyed token. Direct identifiers and raw
|
||||
source payloads remain outside the analytical exports and public site.
|
||||
|
||||
## Leakage-safe method
|
||||
|
||||
Every feature exists before the target episode begins. The final model uses
|
||||
vehicle age, prior episode and attempt counts, time since prior history, prior
|
||||
non-pass rate, prior first/final outcomes, previously observed make/model,
|
||||
public county context, and season.
|
||||
|
||||
The model excludes the target attempt's outcome and diagnostics, later attempts
|
||||
from the target episode, station or technician information, future records,
|
||||
full-history leakage, direct or pseudonymous identifiers, and preprocessing
|
||||
learned from later partitions. DMV enrichment and rich current-test OBD fields
|
||||
are not part of the final model.
|
||||
|
||||
The chronology is fixed:
|
||||
|
||||
| Period | Role in the private 10,000-vehicle development sample |
|
||||
| --- | --- |
|
||||
| 2010-2015 | Historical context only |
|
||||
| 2016-2022 | Fit preprocessing and model parameters |
|
||||
| 2023 | Select logistic regularization and compare fixed candidates |
|
||||
| 2024 | Fit Platt calibration and inspect calibration behavior |
|
||||
| 2025 | One-time development-sample holdout |
|
||||
| 2026 partial | Shadow context only |
|
||||
|
||||
Random row splitting is prohibited. A separate set of private vehicle buckets
|
||||
is never used for fitting and serves as a stress test.
|
||||
|
||||
## Model comparison and decision
|
||||
|
||||
The declared comparison includes training prevalence, the literal previous-
|
||||
episode outcome, regularized logistic regression, and histogram gradient
|
||||
boosting. Logistic regression uses `C=0.03`; Platt scaling is fit on the 2024
|
||||
sample partition. The tree exists only to test whether nonlinearity provides a
|
||||
meaningful development gain.
|
||||
|
||||
The table reports non-audit rows from the private, page-sampled 10,000-vehicle
|
||||
development cohort. It is not population performance.
|
||||
|
||||
| Sample evaluation | Model role | PR-AUC | Brier | ROC-AUC | Top-10% precision |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| 2023 tune | Prevalence baseline | 0.125 | 0.1092 | 0.500 | 0.125 |
|
||||
| 2023 tune | Previous-episode baseline | 0.166 | 0.1723 | 0.582 | 0.285 |
|
||||
| 2023 tune | Logistic before calibration | **0.282** | **0.1012** | **0.707** | **0.315** |
|
||||
| 2025 one-time holdout | Calibrated logistic, final | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| 2025 one-time holdout | Calibrated boosted tree, benchmark | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
|
||||
On the 2023 sample partition, logistic regression more than doubled the
|
||||
prevalence baseline's PR-AUC and raised precision among the highest-scored 10%
|
||||
from 12.5% to 31.5%. On the one-time 2025 sample holdout, calibrated logistic
|
||||
also exceeded the simple baselines and the boosted-tree benchmark on PR-AUC and
|
||||
Brier score.
|
||||
|
||||
The explicit 2025 gate was opened once after model specifications were frozen
|
||||
from pre-2025 data. No model was changed in response. That evaluation is a
|
||||
development holdout, not a pristine future test, and no further 2025-informed
|
||||
tuning is allowed.
|
||||
|
||||
The final decision is therefore straightforward: keep calibrated logistic
|
||||
regression. It satisfies the development objective with a simpler, more
|
||||
explainable form, while the nonlinear benchmark did not show a gain that
|
||||
justified extra complexity.
|
||||
|
||||
## Static dashboard
|
||||
|
||||
The presentation-ready dashboard is a dependency-free static site with four
|
||||
views:
|
||||
|
||||
1. **Overview** shows rounded support, sample outcome patterns, vehicle-age
|
||||
patterns, and county feed coverage.
|
||||
2. **Sample cohorts** shows only supported make/model sample aggregates; it does
|
||||
not issue reliability ratings or recommendations.
|
||||
3. **Model & benchmark** identifies calibrated logistic regression as final and
|
||||
the boosted tree as benchmark-only. It shows pre-2025, 2024 calibration-fit
|
||||
checks—not the opened 2025 holdout comparison.
|
||||
4. **Data & methods** explains the target, chronology, evidence status,
|
||||
limitations, and privacy boundary.
|
||||
|
||||
The finished dashboard contains no vehicle lookup, personal input form, or
|
||||
row-level prediction service.
|
||||
|
||||
The dashboard loads purpose-built JSON rather than private analytical rows. It
|
||||
checks schemas, cross-file consistency, publication flags, and checksums. If the
|
||||
bundle fails validation, the interface shows an unavailable state instead of
|
||||
fallback estimates.
|
||||
|
||||
Every published cell requires minimum support for episodes, distinct vehicles,
|
||||
and both binary outcomes. Supports are rounded and suppressed rows are omitted
|
||||
entirely. The checked-in assets use only pre-2025 sample aggregates and
|
||||
diagnostics; the one-time 2025 values in this report are not in the browser
|
||||
bundle.
|
||||
|
||||
## Privacy and release boundary
|
||||
|
||||
The public boundary is intentionally narrow:
|
||||
|
||||
```text
|
||||
private read-only source
|
||||
-> private local extraction and model pipeline
|
||||
-> suppression-reviewed aggregate JSON
|
||||
-> dashboard-only static Bolt project
|
||||
```
|
||||
|
||||
The public dashboard and presentation materials contain no VIN, plate, ZIP,
|
||||
station, technician identifier, private vehicle token, raw JSON, credential,
|
||||
operational record, or row-level prediction. The browser never connects to the
|
||||
source database.
|
||||
|
||||
Bolt publication must use a separate project or repository containing only the
|
||||
contents of `dashboard/`, with `index.html` at its root. The full source
|
||||
repository must not be imported into Bolt. The reviewed procedure is in
|
||||
[bolt_deployment.md](bolt_deployment.md).
|
||||
|
||||
## Limitations
|
||||
|
||||
- The private page sample is not population-representative.
|
||||
- Results cover returning vehicles in participating feeds, not all Utah
|
||||
vehicles or counties.
|
||||
- Geography, source era, program mix, and time are entangled.
|
||||
- Reject and abort are heterogeneous non-pass outcomes.
|
||||
- The 2025 comparison is a one-time development holdout, not external
|
||||
validation.
|
||||
- Sample calibration and ranking do not establish operational usefulness.
|
||||
- Observed cohort differences are descriptive associations, not causal effects
|
||||
or recommendations.
|
||||
- The absence of protected attributes does not establish fairness.
|
||||
|
||||
## Conclusion
|
||||
|
||||
The prototype answers its development question: a leakage-safe, calibrated
|
||||
logistic regression produces materially better discrimination and probability-error metrics
|
||||
than simple baselines in the private 10,000-vehicle sample, and it performs
|
||||
better than the boosted-tree benchmark in the one-time holdout comparison. The
|
||||
more important product result is the disciplined boundary around that finding:
|
||||
one target, chronological evaluation, transparent model selection, persistent
|
||||
sample labeling, fail-closed aggregate publication, and no row-level exposure.
|
||||
|
||||
This is a completed development prototype—not a population estimate, county
|
||||
ranking, production model, diagnosis, or individual decision tool.
|
||||
|
||||
## Supporting documentation
|
||||
|
||||
- [Project charter](project_charter.md)
|
||||
- [Modeling protocol](modeling_protocol.md)
|
||||
- [Detailed development results](development_results.md)
|
||||
- [Model card](model_card.md)
|
||||
- [Dashboard specification](dashboard_spec.md)
|
||||
- [Private-to-public architecture](architecture.mmd)
|
||||
- [10-minute presentation outline](presentation_outline.md)
|
||||
- [Dashboard demo script](demo_script.md)
|
||||
- [Dashboard-only Bolt deployment](bolt_deployment.md)
|
||||
+118
-154
@@ -1,197 +1,161 @@
|
||||
# Utah Vehicle Health model card
|
||||
|
||||
Last updated: 2026-07-15
|
||||
Last updated: 2026-07-21
|
||||
|
||||
Status: **development prototype; not approved for production or population claims**
|
||||
Status: **final development-prototype model; not approved for production or
|
||||
population claims**
|
||||
|
||||
> Every count, rate, and performance value in this model card comes from the
|
||||
> private, page-sampled 10,000-vehicle development cohort. These are sample
|
||||
> results, not population estimates.
|
||||
|
||||
## Model summary
|
||||
|
||||
Utah Vehicle Health estimates the probability that the first attempt of a
|
||||
returning vehicle's next emissions-inspection episode will be a non-pass. The
|
||||
current selected model is a regularized logistic regression followed by Platt
|
||||
probability calibration. It is an inspection-history model, not a mechanical
|
||||
health, safety, roadworthiness, or legal-compliance model.
|
||||
Utah Vehicle Health estimates whether the first attempt of a returning
|
||||
vehicle's next emissions-inspection episode will be a non-pass. The final model
|
||||
is regularized logistic regression (`C=0.03`) followed by Platt probability
|
||||
calibration. Histogram gradient boosting is retained only as a benchmark.
|
||||
|
||||
This model card summarizes the current page-sampled development run. The full
|
||||
research and product contracts are in the [project charter](project_charter.md),
|
||||
[modeling protocol](modeling_protocol.md), and
|
||||
[dashboard specification](dashboard_spec.md).
|
||||
This is an inspection-history model. It is not a model of overall mechanical
|
||||
health, safety, roadworthiness, or legal compliance.
|
||||
|
||||
## Intended use and prohibited use
|
||||
## Intended use
|
||||
|
||||
Intended uses are to validate the data-engineering and modeling pipeline,
|
||||
compare leakage-safe model candidates, study aggregate patterns, and support a
|
||||
clearly marked development dashboard preview.
|
||||
The development model supports:
|
||||
|
||||
Do not use the model to:
|
||||
- validating leakage-safe longitudinal feature engineering;
|
||||
- comparing a transparent final model with simple and nonlinear benchmarks;
|
||||
- explaining calibration and chronological evaluation; and
|
||||
- powering aggregate, suppression-reviewed development-preview diagnostics.
|
||||
|
||||
- make decisions about an individual vehicle, owner, registration, inspection,
|
||||
or station;
|
||||
- diagnose a vehicle, guarantee an inspection result, or infer safety or
|
||||
roadworthiness;
|
||||
- rank or penalize people, counties, programs, or inspection stations;
|
||||
- make causal claims from observed associations; or
|
||||
- report statewide prevalence, county rankings, or production performance from
|
||||
the current development sample.
|
||||
Do not use it to:
|
||||
|
||||
The public estimator remains disabled because no privacy-reviewed prediction
|
||||
lookup has been approved.
|
||||
- decide anything about an individual vehicle, owner, registration, inspection,
|
||||
county, program, station, or technician;
|
||||
- diagnose a vehicle or guarantee an inspection outcome;
|
||||
- rank counties, programs, stations, vehicle cohorts, or people;
|
||||
- make causal claims from observed sample associations;
|
||||
- report statewide/county prevalence or production performance; or
|
||||
- serve a prediction lookup or row-level score.
|
||||
|
||||
## Prediction unit and target
|
||||
|
||||
The prediction is made immediately before a new inspection episode begins. An
|
||||
episode groups consecutive attempts no more than 30 days apart, and only its
|
||||
first attempt is the supervised target. A vehicle must have at least one prior
|
||||
completed episode to enter the returning-vehicle cohort.
|
||||
The prediction is made immediately before a new inspection episode begins.
|
||||
Attempts no more than 30 days apart form one episode. The supervised target is
|
||||
the episode's first attempt, and a vehicle must have at least one prior completed
|
||||
episode.
|
||||
|
||||
The binary target is:
|
||||
The binary target is recognized pass (`0`) versus recognized fail, reject, or
|
||||
abort (`1`, collectively **non-pass**). Blank, null, and unrecognized results
|
||||
remain unlabeled. Reject and abort may reflect process/readiness conditions, so
|
||||
non-pass must not be paraphrased as mechanical failure.
|
||||
|
||||
- `0`: recognized pass;
|
||||
- `1`: recognized fail, reject, or abort; and
|
||||
- unlabeled: blank, null, or unrecognized results.
|
||||
A recognized overall result is preferred. The documented older Utah County OBD
|
||||
exception may fill a blank overall result under a narrow source/program/test
|
||||
contract. That value is a binary proxy only. Its provenance remains private
|
||||
audit metadata and is not a predictor. See the
|
||||
[modeling protocol](modeling_protocol.md#binary-label-contract).
|
||||
|
||||
A recognized overall result is preferred. For the older Utah County OBD feed,
|
||||
a controlled OBD result may fill a blank overall result only under the narrow
|
||||
source/program/test contract documented in the
|
||||
[modeling protocol](modeling_protocol.md#source-specific-label-contract). That
|
||||
proxy is approved only for binary pass versus non-pass; it must not support a
|
||||
four-class interpretation. Label provenance is private audit metadata and is
|
||||
not a predictor.
|
||||
## Development data
|
||||
|
||||
## Development data and representativeness
|
||||
The extraction page-samples inspection records and retrieves complete histories
|
||||
for up to 10,000 vehicles. After validation, the private run retained 9,996
|
||||
vehicle tokens, 83,552 source events, 69,588 episodes, and 44,659 eligible
|
||||
returning targets. These sample counts describe pipeline reconciliation only;
|
||||
they do not imply population coverage.
|
||||
|
||||
The current extract samples inspection-table pages and then retrieves complete
|
||||
histories for up to 10,000 vehicles. This over-represents vehicles with more
|
||||
inspection records. The run contains 83,552 retained source events, 69,588
|
||||
episodes, and 44,659 eligible returning targets, but those counts do not make
|
||||
the sample population-representative.
|
||||
The sampling design over-represents vehicles with more inspection records. Feed
|
||||
coverage changes by source and time and does not include all 29 Utah counties.
|
||||
Accordingly, the model card makes no statewide prevalence claim, no county
|
||||
ranking, and no claim that sample subgroup differences generalize.
|
||||
|
||||
Coverage is limited to participating county/source feeds and changes over time.
|
||||
It does not represent all 29 Utah counties. Source-system transitions, missing
|
||||
years, and partial periods can resemble real changes in risk. See the
|
||||
[data inventory](data_inventory.md) and
|
||||
[development results](development_results.md) for the audited scope.
|
||||
## Features and exclusions
|
||||
|
||||
## Features and leakage exclusions
|
||||
The final model uses only pre-episode inspection-history information: vehicle
|
||||
age, prior episode/attempt counts, time since prior history, prior non-pass rate,
|
||||
prior first/final outcomes, previously observed make/model, public county
|
||||
context, and season.
|
||||
|
||||
The selected model uses information available before the target episode:
|
||||
The model excludes current-test results and diagnostics, later attempts, station
|
||||
and technician information, direct/pseudonymous identifiers, future records,
|
||||
full-history leakage, DMV enrichment, and preprocessing learned from evaluation
|
||||
periods. It does not use VIN, plate, ZIP, private token, raw JSON, credentials,
|
||||
or operational records. The complete contract is in
|
||||
[modeling_protocol.md](modeling_protocol.md#leakage-and-privacy-exclusions).
|
||||
|
||||
- vehicle age;
|
||||
- prior episode and attempt counts;
|
||||
- days since the prior episode and prior adverse outcome;
|
||||
- prior non-pass rate;
|
||||
- prior first and final outcomes;
|
||||
- previously observed make and model;
|
||||
- public county; and
|
||||
- target season.
|
||||
## Chronological evaluation
|
||||
|
||||
Categorical mappings and all preprocessing are fit on training data only. The
|
||||
current model does not yet include DMV enrichment or rich same-test OBD fields.
|
||||
|
||||
Excluded inputs include direct or pseudonymous identifiers, station
|
||||
information, the target attempt's result or diagnostics, later attempts in the
|
||||
target episode, future records, full-history aggregates, label provenance, and
|
||||
preprocessing learned from evaluation periods. The complete exclusion contract
|
||||
is in [modeling_protocol.md](modeling_protocol.md#leakage-exclusions).
|
||||
|
||||
## Chronology and holdout status
|
||||
|
||||
The fixed temporal design is:
|
||||
|
||||
| Period | Role |
|
||||
| Period | Development-sample role |
|
||||
| --- | --- |
|
||||
| 2010-2015 | Historical context only |
|
||||
| 2016-2022 | Fit preprocessing and models |
|
||||
| 2023 | Select hyperparameters and compare candidates |
|
||||
| 2024 | Fit probability calibration and inspect calibration behavior |
|
||||
| 2025 | One-time development holdout |
|
||||
| 2026 partial | Shadow monitoring only |
|
||||
| 2016-2022 | Fit preprocessing and logistic parameters |
|
||||
| 2023 | Select regularization and compare fixed candidates |
|
||||
| 2024 | Fit Platt scaling and inspect calibration behavior |
|
||||
| 2025 | One-time development-sample holdout |
|
||||
| 2026 partial | Shadow context only |
|
||||
|
||||
During live verification, the explicit 2025 gate was opened once after both
|
||||
candidate specifications and the tree search grid had already been fixed from
|
||||
pre-2025 data. No model was changed in response. The already-fixed logistic
|
||||
model was then evaluated once for a direct comparison. Consequently, this
|
||||
page-sampled 2025 cohort is no longer a pristine unseen test, and no further
|
||||
2025-informed tuning is permitted. Any complete-data 2025 analysis must be
|
||||
described as confirmatory. The detailed audit trail is in
|
||||
[development_results.md](development_results.md#one-time-development-holdout).
|
||||
The explicit 2025 gate was opened once after specifications were frozen from
|
||||
pre-2025 data. No model was changed in response. This makes 2025 a one-time
|
||||
development holdout, not a pristine future test. No further 2025-informed tuning
|
||||
is permitted.
|
||||
|
||||
## Candidate comparison and selected model
|
||||
## Development-sample performance
|
||||
|
||||
The candidates were training prevalence, the literal previous-episode outcome,
|
||||
regularized logistic regression, and histogram gradient boosting. Results below
|
||||
use non-audit development rows.
|
||||
The following values use non-audit rows from the **private 10,000-vehicle
|
||||
development sample**. They are **not population performance estimates**.
|
||||
|
||||
| Evaluation | Model | PR-AUC | Brier | ROC-AUC | Top-10% precision |
|
||||
| Sample evaluation | Model role | PR-AUC | Brier | ROC-AUC | Top-10% precision |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| 2023 tuning | Training prevalence | 0.125 | 0.1092 | 0.500 | 0.125 |
|
||||
| 2023 tuning | Previous episode | 0.166 | 0.1723 | 0.582 | 0.285 |
|
||||
| 2023 tuning | Logistic | **0.282** | **0.1012** | **0.707** | **0.315** |
|
||||
| 2025 one-time holdout | Logistic + Platt | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| 2025 one-time holdout | Gradient boosting + Platt | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
| 2023 tune | Training-prevalence baseline | 0.125 | 0.1092 | 0.500 | 0.125 |
|
||||
| 2023 tune | Previous-episode baseline | 0.166 | 0.1723 | 0.582 | 0.285 |
|
||||
| 2023 tune | Logistic before calibration | **0.282** | **0.1012** | **0.707** | **0.315** |
|
||||
| 2025 one-time holdout | Calibrated logistic, final | **0.261** | **0.1011** | **0.693** | **0.312** |
|
||||
| 2025 one-time holdout | Calibrated boosted tree, benchmark | 0.238 | 0.1024 | 0.690 | 0.292 |
|
||||
|
||||
The selected logistic model uses `C=0.03`. It satisfied explicit convergence
|
||||
and finite-value checks. Platt scaling was fit on the 2024 calibration cohort;
|
||||
on that same cohort, PR-AUC was 0.257 and Brier score was 0.0936. Those 2024
|
||||
values are calibration diagnostics, not independent final performance.
|
||||
Platt scaling was fit on the 2024 development-sample calibration partition. Its
|
||||
same-partition PR-AUC of 0.257 and Brier score of 0.0936 are calibration
|
||||
diagnostics, not independent final performance.
|
||||
|
||||
The logistic model remains selected because it beat both simple baselines and
|
||||
the more complex tree on the one-time development holdout while remaining more
|
||||
transparent. The separate never-fit vehicle audit also supported the logistic
|
||||
model, but the development sample is too limited for final generalization or
|
||||
fairness claims.
|
||||
The final logistic model was retained because it beat the two simple baselines
|
||||
on headline development metrics and outperformed the more complex tree in the
|
||||
one-time sample holdout. The benchmark did not justify added complexity.
|
||||
|
||||
## Coverage, fairness, and privacy limitations
|
||||
The never-fit private-vehicle stress-test subset contained 480 eligible 2025
|
||||
sample episodes from 450 vehicles; calibrated logistic PR-AUC was 0.253 and
|
||||
Brier score was 0.0859. These are also private-sample diagnostics, not
|
||||
population or external-validation results.
|
||||
|
||||
- The model applies only to returning vehicles with recognizable labels and
|
||||
sufficient prior history; cold-start behavior is not established.
|
||||
- Geography and source era are entangled. Performance may shift when a feed,
|
||||
county program, vehicle mix, or label process changes.
|
||||
- Reject and abort outcomes can reflect readiness or process issues rather than
|
||||
mechanical failure.
|
||||
- No protected attributes are modeled, but their absence does not establish
|
||||
fairness. Subgroup sample sizes, errors, and calibration still require review.
|
||||
- Make/model normalization and complete-data subgroup analysis are unfinished.
|
||||
- The current probability calibration has not yet received independent,
|
||||
vehicle-clustered uncertainty analysis or external validation.
|
||||
## Limitations
|
||||
|
||||
Private analytical artifacts remain local. Public assets contain only reviewed,
|
||||
rounded aggregates that satisfy episode, distinct-vehicle, and binary-class
|
||||
suppression thresholds. Direct identifiers, private linkage values, raw rows,
|
||||
and row-level predictions are outside the public contract. See
|
||||
[dashboard_spec.md](dashboard_spec.md#publication-controls).
|
||||
- Results apply only to the sampled returning-vehicle cohort with recognizable
|
||||
labels and sufficient prior history.
|
||||
- Page sampling is not population-representative.
|
||||
- Source, program, time, and geography are entangled.
|
||||
- Reject and abort are heterogeneous non-pass outcomes.
|
||||
- Make/model aliases and incomplete feed coverage can distort cohorts.
|
||||
- The one-time 2025 holdout is not an external validation dataset.
|
||||
- No protected attributes are modeled, but that does not establish fairness.
|
||||
- Sample calibration does not establish production calibration.
|
||||
|
||||
## Monitoring
|
||||
## Privacy and public release
|
||||
|
||||
Before any deployment, monitoring must cover source-level volume, label
|
||||
recognition, outcome prevalence, source/program mix, missingness, unseen
|
||||
categories, score distributions, PR-AUC, Brier score, and calibration. Reviews
|
||||
must explicitly separate the Salt Lake source transition, newer rich feeds, DMV
|
||||
coverage gaps, and the partial 2026 period. Alert thresholds and a response plan
|
||||
remain to be defined.
|
||||
Private analytical artifacts remain local and Git-ignored. Dashboard files
|
||||
contain only rounded aggregate cells that pass episode, distinct-vehicle, and
|
||||
binary-class suppression. The browser validates publication flags and checksums
|
||||
and fails closed.
|
||||
|
||||
No VIN, plate, ZIP, station, technician identifier, private token, raw JSON,
|
||||
credential, operational record, or row-level prediction belongs in a public
|
||||
asset. The finished dashboard has no vehicle-level prediction service. See the
|
||||
[dashboard specification](dashboard_spec.md#suppression-and-privacy-controls).
|
||||
|
||||
## Reproducibility
|
||||
|
||||
The pipeline uses fixed temporal boundaries and a fixed random seed. Python
|
||||
dependencies are pinned in [requirements.txt](../requirements.txt). Private
|
||||
manifests record input lineage, software/model configuration, convergence, row
|
||||
reconciliation, and checksums without publishing private paths or values. SQL
|
||||
transforms and validation queries are versioned under [sql](../sql), model
|
||||
runners are under [scripts](../scripts), and automated checks are under
|
||||
[tests](../tests). Normal training commands leave the 2025 evaluation gate
|
||||
closed unless an explicit flag is supplied.
|
||||
The pipeline uses fixed chronological boundaries and a fixed seed. Private
|
||||
manifests bind input lineage, configuration, convergence checks, row
|
||||
reconciliation, and checksums. Normal training commands keep the 2025 gate
|
||||
closed unless the explicit evaluation flag is supplied.
|
||||
|
||||
## Remaining approval gates
|
||||
|
||||
1. Run a complete, contiguous, bounded extraction and rebuild the frozen
|
||||
pipeline without page-sampling bias.
|
||||
2. Treat complete-data 2025 results as confirmation and reserve a genuinely new
|
||||
period or external dataset for future unseen evaluation.
|
||||
3. Complete subgroup/source-era reporting, vehicle-clustered uncertainty,
|
||||
calibration diagnostics, and fairness review.
|
||||
4. Finish make/model normalization, episode-gap sensitivity checks, and the
|
||||
preregistered inspection-only versus DMV-enhanced ablation.
|
||||
5. Pass privacy and publication review before replacing the development-preview
|
||||
aggregates or enabling any prediction lookup.
|
||||
6. Define monitoring thresholds, ownership, rollback criteria, and a model
|
||||
update policy before operational use.
|
||||
The detailed sample audit is in
|
||||
[development_results.md](development_results.md), and the narrative conclusion
|
||||
is in [final_report.md](final_report.md).
|
||||
|
||||
+95
-103
@@ -1,141 +1,133 @@
|
||||
# Leakage-safe modeling protocol
|
||||
|
||||
## Reporting boundary
|
||||
|
||||
This protocol governs the private, page-sampled 10,000-vehicle development
|
||||
cohort. All resulting counts, rates, comparisons, and performance metrics are
|
||||
**sample results, not population estimates**. County/source reporting is for
|
||||
coverage and drift context only; it must not be turned into population county
|
||||
rankings.
|
||||
|
||||
## Prediction unit
|
||||
|
||||
Score the **first attempt of the next inspection episode** immediately before
|
||||
check-in. Consecutive attempts for the same private vehicle token belong to the
|
||||
same episode when they are no more than 30 days apart. Re-run the analysis with
|
||||
14- and 45-day gaps as sensitivity checks.
|
||||
Estimate the probability that the **first attempt of a returning vehicle's next
|
||||
inspection episode is a non-pass**, immediately before the episode begins.
|
||||
Consecutive attempts for the same private vehicle token form one episode when
|
||||
they are no more than 30 days apart. Rapid retests are not separate targets.
|
||||
|
||||
This prevents rapid fail/retest sequences from dominating the target. Those
|
||||
within-episode attempts belong in the separate failure-to-pass journey analysis.
|
||||
This is the sole research target for the completed prototype. Cold-start,
|
||||
four-class, fail-only, failure-to-pass, and individualized prediction tasks are
|
||||
out of scope.
|
||||
|
||||
## Eligibility
|
||||
|
||||
- Use 2010-2015 events only as historical context; supervised targets begin in
|
||||
2016.
|
||||
- Exclude the four 1990 date outliers.
|
||||
- Require a recognized first-attempt outcome and at least one prior completed
|
||||
episode for the returning-vehicle model.
|
||||
- Keep first-observed vehicles as a separate cold-start cohort.
|
||||
- Exclude the four known 1990 date outliers.
|
||||
- Require a recognized first-attempt binary outcome and at least one prior
|
||||
completed episode.
|
||||
- Keep first-observed vehicles outside the modeled returning-vehicle cohort.
|
||||
- Normalize `slc` and `slco` to Salt Lake County for geography while preserving
|
||||
source era for drift reporting.
|
||||
- Deduplicate exact uploads and quarantine shared/test/placeholder identifiers
|
||||
using preregistered rules for impossible conflicts or extreme activity.
|
||||
- Do not require a DMV match; retain explicit match and staleness indicators.
|
||||
source era for diagnostics.
|
||||
- Deduplicate exact uploads and quarantine shared, test, or placeholder
|
||||
identifiers under preregistered conflict/activity rules.
|
||||
|
||||
Every exclusion must appear in a cohort-flow report.
|
||||
Every exclusion appears in the private cohort-flow audit.
|
||||
|
||||
## Source-specific label contract
|
||||
## Binary label contract
|
||||
|
||||
The normalized target prefers a recognized `overall_result` for every source.
|
||||
The older Utah County feed is the sole exception: when its overall result is
|
||||
blank, `obd_result` may supply the binary target only when the source is
|
||||
`utah`, the program is `obd`, the test type is `OBD`, and the controlled value
|
||||
is pass, fail, reject, or abort. `B`, blank, TSI, `other/C`, and unknown values
|
||||
remain unlabeled. Every target retains `target_outcome_label_source` as private
|
||||
audit metadata, and that field is never a predictor.
|
||||
The target is `0` for a recognized pass and `1` for recognized fail, reject, or
|
||||
abort. Blank, null, and unrecognized values remain unlabeled rather than being
|
||||
treated as passes.
|
||||
|
||||
This is a binary pass-versus-non-pass proxy, not a four-class substitution.
|
||||
Across non-Utah feeds where both fields are recognized, an aggregate audit found
|
||||
99.30% agreement on pass versus non-pass, while the fail/reject distinction was
|
||||
not interchangeable. Utah program rules also distinguish a readiness rejection
|
||||
from a failed inspection. Future four-class analysis must therefore require
|
||||
`target_outcome_label_source = 'overall_result'`. See the official
|
||||
[Utah inspection requirements](https://dmv.utah.gov/register/inspections/) and
|
||||
[program definition of rejection](https://www.utah.gov/pmn/files/1155003.pdf).
|
||||
The normalized target prefers a recognized `overall_result`. The older Utah
|
||||
County feed is the sole exception: when its overall result is blank,
|
||||
`obd_result` may supply the binary label only when source is `utah`, program is
|
||||
`obd`, test type is `OBD`, and the controlled value is pass, fail, reject, or
|
||||
abort. `B`, blank, TSI, `other/C`, and unknown values remain unlabeled. Private
|
||||
label provenance is retained for audit and is never a predictor or public
|
||||
field.
|
||||
|
||||
This exception is a binary pass/non-pass proxy. It does not justify interpreting
|
||||
fail, reject, and abort as interchangeable mechanical conditions.
|
||||
|
||||
## Point-in-time features
|
||||
|
||||
Every historical window ends strictly before the target episode:
|
||||
Every feature window ends strictly before the target episode. The final
|
||||
inspection-history feature set includes:
|
||||
|
||||
- vehicle age;
|
||||
- prior episode and attempt counts;
|
||||
- previous episode first/final outcome;
|
||||
- expanding and trailing prior pass/fail/reject/abort counts and rates;
|
||||
- days since the previous episode and prior adverse result;
|
||||
- attempts required in prior episodes;
|
||||
- vehicle age and canonical make/model from prior information;
|
||||
- physical county and target month/season;
|
||||
- latest DMV record dated before the target, registration count/recency, fuel,
|
||||
and DMV match/staleness indicators; and
|
||||
- missingness indicators.
|
||||
- days since the prior episode and prior adverse outcome;
|
||||
- prior non-pass rate;
|
||||
- prior episode first and final outcomes;
|
||||
- previously observed make and model;
|
||||
- public county context; and
|
||||
- target season.
|
||||
|
||||
The stable core model is inspection-only. The DMV-enhanced model is an explicit
|
||||
ablation because DMV history ends in March 2024 and is nearly absent in 2021.
|
||||
The final prototype does not use DMV enrichment or rich current-test OBD data.
|
||||
|
||||
## Leakage exclusions
|
||||
## Leakage and privacy exclusions
|
||||
|
||||
The MVP must not use:
|
||||
The model and public product exclude:
|
||||
|
||||
- the target attempt's overall/OBD result, result reason, DTC count, MIL or
|
||||
readiness values, PIDs, visual checks, measurements, certificate, or
|
||||
calibration fields;
|
||||
- the target attempt's result, reason, DTC count, MIL/readiness state, PIDs,
|
||||
visual checks, measurements, certificate, or calibration fields;
|
||||
- later attempts or eventual outcome from the target episode;
|
||||
- station or station-level outcome statistics;
|
||||
- raw VIN, private token, plate, ZIP, or other identifiers as features;
|
||||
- target-row vehicle attributes when a prior/static source is available;
|
||||
- future DMV records;
|
||||
- full-history aggregates; or
|
||||
- preprocessing, category mappings, target encodings, or imputation learned
|
||||
from validation/test data.
|
||||
- station, technician, or station-level outcome information;
|
||||
- raw VIN, private vehicle token, plate, ZIP, exact address, or another direct
|
||||
or pseudonymous identifier as a feature;
|
||||
- future records, full-history aggregates, or target-row information that was
|
||||
not available before the episode;
|
||||
- preprocessing, category mappings, imputation, or calibration learned from a
|
||||
later partition; and
|
||||
- raw JSON, credentials, operational records, and row-level predictions in any
|
||||
public artifact.
|
||||
|
||||
Timestamp ties must be resolved before lag/window calculations. Cumulative
|
||||
windows end at the preceding event or episode.
|
||||
Timestamp ties are resolved before lag/window calculations. Cumulative windows
|
||||
end at the preceding event or episode.
|
||||
|
||||
## Fixed evaluation timeline
|
||||
## Frozen chronology
|
||||
|
||||
| Partition | Target dates | Purpose |
|
||||
| Partition | Target dates | Role in the private 10,000-vehicle development sample |
|
||||
| --- | --- | --- |
|
||||
| Historical context | 2010-2015 | Lag features only |
|
||||
| Train | 2016-2022 | Fit preprocessing and models |
|
||||
| Tune | 2023 | Hyperparameters and selection |
|
||||
| Calibrate | 2024 | Probability calibration and thresholds |
|
||||
| Locked test | 2025 | Final reported performance |
|
||||
| Shadow drift | 2026-01-01 to 2026-06-22 | Monitoring only |
|
||||
| Train | 2016-2022 | Fit preprocessing and model parameters |
|
||||
| Tune | 2023 | Select logistic regularization; compare fixed candidates |
|
||||
| Calibrate | 2024 | Fit Platt scaling and inspect calibration behavior |
|
||||
| One-time holdout | 2025 | Development comparison after specifications were frozen |
|
||||
| Shadow | 2026-01-01 to 2026-06-22 | Partial-period monitoring context only |
|
||||
|
||||
The page-sampled development extract's 2025 gate was opened once during live
|
||||
verification after both candidate specifications were fixed. It is therefore
|
||||
a one-time development holdout, not a pristine future test. No model change was
|
||||
made from it; a later complete-data 2025 run is confirmatory. See
|
||||
[development_results.md](development_results.md) for the audit trail. The table
|
||||
continues to define the frozen chronology for a complete extraction.
|
||||
The 2025 gate was opened once during live verification. It is no longer a
|
||||
pristine unseen test, no 2025-informed tuning is permitted, and every reported
|
||||
2025 value must say “one-time development-sample holdout.” A later complete-data
|
||||
2025 analysis would be confirmatory, not a new final test.
|
||||
|
||||
Repeated vehicles may cross ordinary time partitions because returning-vehicle
|
||||
prediction is the deployment scenario. Separately reserve 10% of keyed vehicle
|
||||
buckets as a never-fit VIN audit and report its 2025 performance as an unseen-
|
||||
vehicle stress test. Never use random row splitting.
|
||||
Repeated vehicles may cross chronological partitions because returning-vehicle
|
||||
prediction is the intended scenario. Separately held-out private vehicle buckets
|
||||
form a never-fit stress test. Random row splitting is prohibited.
|
||||
|
||||
## Baselines and candidate model
|
||||
## Models and decision rule
|
||||
|
||||
1. Training prevalence
|
||||
2. Repeat the previous episode's first outcome
|
||||
3. Regularized logistic regression with age splines and one-hot categoricals
|
||||
4. A boosted-tree model
|
||||
5. Inspection-only versus inspection-plus-DMV ablation
|
||||
1. Training prevalence: probability baseline.
|
||||
2. Previous episode's first outcome: literal history baseline.
|
||||
3. Regularized logistic regression: selected using the 2023 sample partition,
|
||||
then Platt-calibrated on the 2024 sample partition; **the final model**.
|
||||
4. Histogram gradient boosting with Platt calibration: fixed nonlinear
|
||||
**benchmark only**.
|
||||
|
||||
Keep validation and test sets at natural prevalence. If pass rows are sampled
|
||||
for training, preserve sampling probabilities and recalibrate on the untouched
|
||||
2024 partition.
|
||||
The benchmark does not power a separate dashboard experience. Added complexity
|
||||
would require a clear, predeclared improvement to displace the logistic model;
|
||||
the one-time 2025 sample comparison did not show that improvement.
|
||||
|
||||
## Metrics
|
||||
## Metrics and interpretation
|
||||
|
||||
Headline binary metrics:
|
||||
Headline metrics are non-pass PR-AUC and Brier score. Log loss, ROC-AUC, and
|
||||
precision at the highest-scored 10% are supporting diagnostics. Calibration
|
||||
values computed on the 2024 partition used to fit Platt scaling are calibration
|
||||
diagnostics, not independent final performance.
|
||||
|
||||
- non-pass PR-AUC;
|
||||
- Brier score and log loss;
|
||||
- calibration intercept, slope, and reliability curve;
|
||||
- precision, recall, and lift at fixed review capacities; and
|
||||
- ROC-AUC as secondary context.
|
||||
|
||||
Use vehicle-clustered bootstrap confidence intervals. Report results by county,
|
||||
source era, vehicle-age band, prior outcome, fuel, history depth, and DMV
|
||||
match/staleness. Four-class analysis adds class-specific and macro PR-AUC,
|
||||
multiclass log loss/Brier score, calibration, and a confusion matrix.
|
||||
|
||||
## Drift contract
|
||||
|
||||
Monitor monthly/source-level volume, label recognition, outcome prevalence,
|
||||
blank rate, source/program mix, missingness, DMV staleness, unseen categories,
|
||||
prediction distribution, PR-AUC, Brier score, and calibration. Treat the
|
||||
`slc`→`slco` transition, new rich feeds, DMV 2021 gap, DMV 2024 endpoint, and
|
||||
partial 2026 period as explicit stress cases rather than ordinary random drift.
|
||||
All tables and charts must state that they use the private 10,000-vehicle
|
||||
development sample and are not population estimates. Report sample support with
|
||||
every subgroup result. Do not describe observed differences as causal, do not
|
||||
claim generalization to all Utah vehicles, and do not publish county rankings.
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
# Utah Vehicle Health: 10-minute presentation outline
|
||||
|
||||
## Presentation rule
|
||||
|
||||
Keep this sentence visible on every results or dashboard slide:
|
||||
|
||||
> Private 10,000-vehicle development sample; sample results, not population
|
||||
> estimates.
|
||||
|
||||
Do not show county rankings, imply statewide coverage, or describe non-pass as
|
||||
mechanical failure. Refer to calibrated logistic regression as the **final
|
||||
model** and histogram gradient boosting as the **benchmark**.
|
||||
|
||||
## Timed outline
|
||||
|
||||
| Time | Slide / action | Core message |
|
||||
| --- | --- | --- |
|
||||
| 0:00-0:40 | 1. Title and boundary | One focused prediction question; development prototype, not a population study or decision tool |
|
||||
| 0:40-1:25 | 2. Research question | Returning vehicle, next episode, first attempt, binary non-pass |
|
||||
| 1:25-2:15 | 3. Development data | Private page-sampled 10,000-vehicle histories; complete sampled histories but nonrepresentative selection |
|
||||
| 2:15-3:10 | 4. Leakage-safe design | 30-day episodes, only pre-episode history, chronological split, never random rows |
|
||||
| 3:10-4:20 | 5. Model comparison | Logistic final versus prevalence/previous-outcome baselines and boosted-tree benchmark |
|
||||
| 4:20-4:50 | 6. Model decision | Calibrated logistic wins the declared sample comparison and is easier to explain |
|
||||
| 4:50-7:35 | Live dashboard demo | Overview, Sample cohorts, Model & benchmark, Data & methods |
|
||||
| 7:35-8:30 | 7. Privacy architecture | Private pipeline to suppressed aggregate JSON to dashboard-only Bolt project |
|
||||
| 8:30-9:20 | 8. Limitations | Sampling, feed coverage, source/time confounding, heterogeneous non-pass, one-time holdout |
|
||||
| 9:20-10:00 | 9. Close | Prototype is complete and honest about what it can—and cannot—claim |
|
||||
|
||||
Total: **10:00**
|
||||
|
||||
## Slide notes
|
||||
|
||||
### 1. Title and boundary — 40 seconds
|
||||
|
||||
**Title:** Utah Vehicle Health
|
||||
|
||||
**Subtitle:** Predicting a returning vehicle's next-episode first-attempt
|
||||
non-pass outcome
|
||||
|
||||
Say:
|
||||
|
||||
> This is a presentation-ready development prototype built from a private,
|
||||
> page-sampled 10,000-vehicle cohort. Every result I show is sample-based, not a
|
||||
> population estimate. The project does not rank counties or diagnose vehicles.
|
||||
|
||||
### 2. Research question — 45 seconds
|
||||
|
||||
Show the question:
|
||||
|
||||
> Using only information available before an inspection episode begins, how
|
||||
> well can a calibrated logistic regression estimate whether a returning
|
||||
> vehicle's next episode will have a first-attempt non-pass outcome?
|
||||
|
||||
Explain that attempts separated by no more than 30 days form one episode. This
|
||||
keeps rapid fail/retest sequences from becoming repeated target rows. Non-pass
|
||||
combines fail, reject, and abort; blanks remain unlabeled.
|
||||
|
||||
### 3. Development data — 50 seconds
|
||||
|
||||
Show a compact flow:
|
||||
|
||||
```text
|
||||
sample up to 10,000 vehicles -> retrieve their histories -> build episodes
|
||||
-> create 44,659 eligible returning targets
|
||||
```
|
||||
|
||||
Say that 9,996 private tokens remained after validation, producing 83,552 source
|
||||
events and 69,588 episodes in the private sample. Immediately repeat that these
|
||||
are reconciliation counts, not Utah totals. Page sampling over-represents
|
||||
vehicles with more inspection records, and participating feeds do not cover all
|
||||
29 counties.
|
||||
|
||||
### 4. Leakage-safe design — 55 seconds
|
||||
|
||||
Show the chronology:
|
||||
|
||||
```text
|
||||
2010-15 context | 2016-22 train | 2023 tune | 2024 calibrate
|
||||
| 2025 one-time holdout | 2026 partial shadow
|
||||
```
|
||||
|
||||
Explain that every feature window ends before the target episode. The model uses
|
||||
prior outcomes, history depth, timing, age, previously observed make/model,
|
||||
public county context, and season. It excludes current-test diagnostics, future
|
||||
records, identifiers, stations, technicians, and later attempts.
|
||||
|
||||
Note that the 2025 sample gate was opened once after specifications were frozen;
|
||||
it is a development holdout, not a pristine future test.
|
||||
|
||||
### 5. Model comparison — 70 seconds
|
||||
|
||||
Use a two-metric table. Label it “Private 10,000-vehicle development sample; not
|
||||
population performance.”
|
||||
|
||||
| 2025 one-time sample holdout | Role | PR-AUC | Brier |
|
||||
| --- | --- | ---: | ---: |
|
||||
| Training prevalence | Baseline | 0.123 | 0.1076 |
|
||||
| Previous episode | Baseline | 0.158 | 0.1731 |
|
||||
| Logistic + Platt | **Final** | **0.261** | **0.1011** |
|
||||
| Boosted tree + Platt | Benchmark | 0.238 | 0.1024 |
|
||||
|
||||
Say that higher PR-AUC and lower Brier are better. Logistic beat both baselines
|
||||
and the boosted-tree benchmark in this one-time sample comparison. Avoid the
|
||||
word “accuracy,” which obscures class imbalance and calibration.
|
||||
|
||||
### 6. Model decision — 30 seconds
|
||||
|
||||
Say:
|
||||
|
||||
> The final model is calibrated logistic regression. The boosted tree is a
|
||||
> benchmark only. It did not improve the declared sample metrics enough to
|
||||
> justify greater complexity, while logistic regression is easier to audit and
|
||||
> explain.
|
||||
|
||||
### Live dashboard demo — 2 minutes 45 seconds
|
||||
|
||||
Follow [demo_script.md](demo_script.md). Keep the development-preview banner in
|
||||
view at the start. Explicitly show calibrated logistic as final, the tree as
|
||||
benchmark-only, and the absence of a vehicle-level prediction service.
|
||||
|
||||
### 7. Privacy architecture — 55 seconds
|
||||
|
||||
Show:
|
||||
|
||||
```text
|
||||
private read-only data -> local modeling -> suppressed aggregate JSON
|
||||
-> dashboard-only Bolt project
|
||||
```
|
||||
|
||||
State that the public bundle has no VINs, plates, ZIPs, stations, technician
|
||||
identifiers, private tokens, raw JSON, credentials, operational records, or
|
||||
row-level predictions. The browser has no database connection and fails closed
|
||||
if its aggregate contract does not validate.
|
||||
|
||||
### 8. Limitations — 50 seconds
|
||||
|
||||
Name the limitations directly:
|
||||
|
||||
- page-sampled cohort is not population-representative;
|
||||
- feeds are partial and change over time;
|
||||
- geography and source era are confounded;
|
||||
- reject/abort do not necessarily mean mechanical failure;
|
||||
- 2025 is a one-time development holdout, not external validation; and
|
||||
- sample associations do not support county rankings or causal conclusions.
|
||||
|
||||
### 9. Close — 40 seconds
|
||||
|
||||
End with:
|
||||
|
||||
> The prototype is complete at the development stage: one precise question, a
|
||||
> leakage-safe timeline, calibrated logistic regression as the final model, a
|
||||
> boosted-tree benchmark, and a privacy-reviewed static dashboard. Its value is
|
||||
> not just the sample performance; it is the discipline of making only the
|
||||
> claims the data and release boundary can support.
|
||||
|
||||
Pause, then invite questions about episode construction, calibration, privacy,
|
||||
or the dashboard—not about county “winners” and “losers,” which this sample
|
||||
cannot establish.
|
||||
|
||||
## Presentation checklist
|
||||
|
||||
- Start the local dashboard before presenting: `node dashboard/server.mjs`.
|
||||
- Open `http://127.0.0.1:4173/#overview` and close unrelated browser tabs.
|
||||
- Use a fresh page load to confirm aggregate validation succeeds.
|
||||
- Keep a screenshot or short recording as a backup, with the sample warning
|
||||
visible.
|
||||
- Never open private data, artifacts, terminal environment variables, database
|
||||
tools, or browser developer tools during the talk.
|
||||
- Do not type or display a VIN, plate, ZIP, station, technician identifier, raw
|
||||
record, or row-level prediction.
|
||||
+81
-93
@@ -1,122 +1,110 @@
|
||||
# Utah Vehicle Health project charter
|
||||
|
||||
## Working title
|
||||
## Prototype status
|
||||
|
||||
**Utah Vehicle Health**
|
||||
|
||||
*What millions of emissions inspections reveal about the next test across the
|
||||
state of Utah county feeds available in this dataset.*
|
||||
**Presentation-ready development prototype.** Every analytical finding and every
|
||||
dashboard value comes from the private, page-sampled 10,000-vehicle development
|
||||
cohort. These values are **sample results, not population estimates**. They must
|
||||
not be used to estimate statewide or county prevalence, create county rankings,
|
||||
or make decisions about an individual vehicle.
|
||||
|
||||
## Product promise
|
||||
|
||||
Explain how vehicle age, type, location, and prior inspection history relate to
|
||||
the chance of passing the first attempt of the next emissions-inspection
|
||||
episode.
|
||||
Utah Vehicle Health explains how information available before an inspection
|
||||
relates to the first-attempt non-pass outcome of a returning vehicle's next
|
||||
inspection episode.
|
||||
|
||||
The product measures **emissions-inspection outcomes**. It must not describe its
|
||||
score as a diagnosis of overall mechanical reliability, roadworthiness, safety,
|
||||
or legal compliance.
|
||||
The product measures an emissions-inspection outcome. It does not measure
|
||||
overall vehicle health, mechanical reliability, safety, roadworthiness, or
|
||||
legal compliance.
|
||||
|
||||
## Primary research question
|
||||
|
||||
> Using only information available before an inspection episode begins, how
|
||||
> accurately and reliably can we estimate whether its first attempt will pass?
|
||||
> well can a calibrated logistic regression estimate whether a returning
|
||||
> vehicle's next episode will have a first-attempt non-pass outcome?
|
||||
|
||||
An episode groups attempts for the same internal vehicle token when the gap
|
||||
between consecutive attempts is 30 days or less. The target is the first attempt
|
||||
of a new episode, not every rapid retest.
|
||||
An episode groups consecutive attempts for the same private vehicle token when
|
||||
the gap is 30 days or less. The target is the first attempt of a new episode,
|
||||
not a rapid retest. A vehicle is eligible only after at least one completed
|
||||
prior episode.
|
||||
|
||||
## Primary target
|
||||
## Binary target
|
||||
|
||||
- `0 — pass`: normalized `PASS` or `P`
|
||||
- `1 — non-pass`: normalized `FAIL`, `F`, `REJECT`, or `ABORT`
|
||||
- Blank, null, and unrecognized results are unlabeled. They remain in chronology
|
||||
and coverage reporting but are excluded as supervised targets.
|
||||
- `0 - pass`: normalized `PASS` or `P`
|
||||
- `1 - non-pass`: normalized `FAIL`, `F`, `REJECT`, or `ABORT`
|
||||
- unlabeled: blank, null, and unrecognized results
|
||||
|
||||
Because reject and abort can reflect process/readiness problems rather than
|
||||
mechanical failure, the project will also report:
|
||||
Reject and abort can reflect process or readiness conditions rather than a
|
||||
mechanical failure. The prototype therefore says **non-pass**, never “vehicle
|
||||
failure,” when referring to the combined target. Multiclass outcome modeling
|
||||
and fail-only sensitivity analysis are outside the finished prototype scope.
|
||||
|
||||
- a four-class pass/fail/reject/abort analysis; and
|
||||
- a fail-versus-pass sensitivity analysis that excludes reject and abort.
|
||||
## Data and product boundary
|
||||
|
||||
## Scope
|
||||
The development cohort contains complete inspection histories for up to 10,000
|
||||
sampled vehicles from participating Utah county/source feeds. Page sampling
|
||||
over-represents vehicles with more inspection records, so neither the cohort nor
|
||||
its dashboard aggregates are population-representative. The feeds do not cover
|
||||
all 29 Utah counties.
|
||||
|
||||
The returning-vehicle MVP uses universal inspection-history fields and prior
|
||||
DMV information where it is point-in-time valid. It covers participating
|
||||
inspection county/source feeds, not all 29 Utah counties.
|
||||
The final model is inspection-history only. DMV enrichment, rich current-test
|
||||
OBD fields, station effects, cold-start prediction, and individualized lookup
|
||||
are outside scope. The static dashboard publishes only rounded, suppressed
|
||||
sample aggregates and pre-2025 development diagnostics.
|
||||
|
||||
The rich OBD/odometer fields are a later, source-specific extension for newer
|
||||
`slco`, `davis`, and `cache` records. They are not part of the statewide-style
|
||||
historical baseline.
|
||||
## Final model and benchmark
|
||||
|
||||
## Deliverables
|
||||
The final model is regularized logistic regression (`C=0.03`) with Platt
|
||||
probability calibration fit on the 2024 development-sample partition. A
|
||||
histogram gradient-boosted tree is retained only as a nonlinear benchmark. It
|
||||
is not a second final model and is not used to drive the product.
|
||||
|
||||
1. A reproducible, read-only extraction and private pseudonymization pipeline.
|
||||
2. A point-in-time episode/feature mart with an auditable exclusion report.
|
||||
3. Prevalence, previous-outcome, and logistic-regression baselines.
|
||||
4. One calibrated boosted-tree model and an inspection-only versus DMV-enhanced
|
||||
ablation.
|
||||
5. A locked temporal evaluation with subgroup and source-era diagnostics.
|
||||
6. Versioned, suppressed public aggregates for a four-page Bolt dashboard.
|
||||
7. A model card and data/methodology page documenting limitations.
|
||||
## Completed deliverables
|
||||
|
||||
## Headline success criteria
|
||||
1. Read-only extraction with private vehicle pseudonymization.
|
||||
2. Leakage-safe episode and point-in-time feature mart.
|
||||
3. Training-prevalence and previous-episode baselines.
|
||||
4. Calibrated logistic regression as the final model.
|
||||
5. Histogram gradient boosting as a benchmark only.
|
||||
6. Chronological development evaluation with an explicit one-time 2025 gate.
|
||||
7. Suppression-reviewed static dashboard assets with fail-closed validation.
|
||||
8. Model card, final report, presentation materials, and dashboard-only Bolt
|
||||
deployment instructions.
|
||||
|
||||
- Beat both the training-prevalence and previous-episode-outcome baselines on
|
||||
2025 PR-AUC and Brier score.
|
||||
- Produce calibrated probabilities, not just class labels.
|
||||
- Report performance by county/source era, vehicle-age band, history depth, and
|
||||
DMV match/staleness.
|
||||
- Reproduce all published charts from versioned sanitized outputs.
|
||||
- Export no VIN, plate, ZIP, station, raw JSON, operational record, or
|
||||
row-level prediction to Bolt.
|
||||
- Suppress public cells with fewer than 100 eligible inspections or fewer than
|
||||
10 observations in an outcome or its complement, with complementary
|
||||
suppression where totals could reveal a hidden cell.
|
||||
## Acceptance criteria
|
||||
|
||||
## Initial feasibility result
|
||||
|
||||
Under label contract v3, the fixed-seed aggregate-only query in
|
||||
`sql/10_episode_cohort_feasibility.sql` produced 8,916 eligible
|
||||
returning-vehicle episodes from 1,730 of 2,000 sampled vehicle histories. The
|
||||
first-attempt non-pass rate was 11.80%, and the median gap from the prior
|
||||
episode was about 372 days. Of those targets, 1,732 use the documented Utah OBD
|
||||
binary proxy because that source leaves its overall-result field blank. This
|
||||
supports both the episode definition and a calibrated binary model. The sample
|
||||
is for pipeline feasibility, not a population estimate.
|
||||
|
||||
The 2025 sample showed a longer median gap and different outcome mix, reinforcing
|
||||
the need for source-era drift reporting and a locked chronological test.
|
||||
|
||||
## Development holdout status
|
||||
|
||||
During live verification, the explicit 2025 gate was opened once on the
|
||||
page-sampled development extract after both candidate specifications had been
|
||||
fixed from pre-2025 data. No test-informed model change was made. The frozen
|
||||
logistic model remains selected, and a later complete-data 2025 analysis will
|
||||
be treated as confirmatory rather than described as a pristine unseen test.
|
||||
The audit trail and results are recorded in
|
||||
[development_results.md](development_results.md).
|
||||
- The prediction unit remains a returning vehicle's next-episode first attempt.
|
||||
- Model features exist before that episode begins.
|
||||
- The calibrated logistic model beats both simple baselines on development-
|
||||
sample PR-AUC and Brier score.
|
||||
- The boosted tree is presented only as a benchmark.
|
||||
- Every displayed value is labeled as a private-sample result and not a
|
||||
population estimate.
|
||||
- County views communicate feed coverage and sample context; they do not claim
|
||||
population rankings or causal county differences.
|
||||
- No VIN, plate, ZIP, station, technician identifier, raw JSON, credential,
|
||||
operational record, private token, or row-level prediction reaches the
|
||||
dashboard or presentation materials.
|
||||
- The dashboard contains no vehicle-level estimator, input form, or prediction
|
||||
service.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Diagnosing an individual vehicle
|
||||
- Certifying that a vehicle will pass
|
||||
- Ranking or accusing inspection stations
|
||||
- Identifying owners or accepting VIN/plate input
|
||||
- Making causal claims about county programs
|
||||
- Treating missing outcomes as passes
|
||||
- Predicting the result of a first-observed vehicle
|
||||
- Diagnosing, certifying, or guaranteeing an individual vehicle outcome
|
||||
- Ranking counties, programs, stations, technicians, owners, or vehicles
|
||||
- Making causal claims about geography, vehicle makes, or inspection programs
|
||||
- Accepting VIN, plate, address, station, or diagnostic inputs
|
||||
- Publishing population prevalence from the development sample
|
||||
- Shipping a production decision service or live database connection
|
||||
- Completing DMV enrichment, four-class modeling, or failure-to-pass journeys
|
||||
|
||||
## Staged build
|
||||
## Evaluation status
|
||||
|
||||
1. **Foundation:** episode definition, label normalization, exclusions, secure
|
||||
extraction, and coverage checks.
|
||||
2. **Baseline:** inspection-history feature mart and transparent baselines.
|
||||
3. **Enrichment:** canonical vehicle dimension and point-in-time DMV features.
|
||||
4. **Modeling:** tree model, calibration, locked test, drift and subgroup audits.
|
||||
5. **Product:** sanitized aggregate export and Bolt MVP.
|
||||
6. **Stretch:** failure-to-pass journeys, multiclass probabilities, and a
|
||||
clearly scoped rich-OBD model.
|
||||
|
||||
See [modeling_protocol.md](modeling_protocol.md) and
|
||||
[dashboard_spec.md](dashboard_spec.md) for the detailed contracts.
|
||||
The explicit 2025 gate was opened once after model specifications were frozen
|
||||
from pre-2025 data. No 2025-informed model change was made. The 2025 values are
|
||||
therefore a one-time **development-sample holdout**, not a pristine future test
|
||||
and not population performance. See [development_results.md](development_results.md)
|
||||
for the audit trail and [modeling_protocol.md](modeling_protocol.md) for the
|
||||
frozen evaluation contract.
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
|
||||
Inventory date: 2026-07-15
|
||||
|
||||
> **Archived planning context:** This shortlist records ideas considered during
|
||||
> discovery; it is not the scope of the finished prototype. DMV enrichment,
|
||||
> multiclass outcomes, station analysis, forecasting, and population/county
|
||||
> rankings below are not current deliverables. The authoritative scope is the
|
||||
> [project charter](project_charter.md), and the completed development story is
|
||||
> in the [final report](final_report.md).
|
||||
|
||||
> **Selected:** Option 1, Utah Vehicle Health. The implementation contract is in
|
||||
> [project_charter.md](project_charter.md).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user