repository is now a presentation-ready development prototype
This commit is contained in:
+74
-105
@@ -1,115 +1,84 @@
|
||||
graph TD
|
||||
%% Data Sources
|
||||
PG[(PostgreSQL<br/>countydata)]
|
||||
DMV[("DMV Data<br/>~18M records<br/>2011-2024")]
|
||||
INSP[("Inspection Data<br/>~19.3M records<br/>2010-2026")]
|
||||
flowchart TD
|
||||
SOURCE[(Private inspection histories<br/>read-only source)]
|
||||
SAMPLE[Private page-sampled histories<br/>up to 10,000 vehicles<br/>keyed vehicle tokens]
|
||||
SOURCE -->|read-only bounded export| SAMPLE
|
||||
|
||||
PG --- DMV
|
||||
PG --- INSP
|
||||
SAMPLE --> STAGE[Normalize labels<br/>deduplicate and validate]
|
||||
STAGE --> EPISODES[Build 30-day episodes<br/>first attempt is the target]
|
||||
EPISODES --> MART[Point-in-time feature mart<br/>returning vehicles only]
|
||||
|
||||
%% Extraction
|
||||
PG -->|"export_history_sample.py<br/>(VIN hashing, page sampling)"| CSV["history_sample_10000.csv.gz<br/>+ manifest"]
|
||||
|
||||
%% Feature Mart Build
|
||||
CSV -->|"build_feature_mart.py"| DUCKDB
|
||||
|
||||
subgraph DuckDB["DuckDB Local Warehouse"]
|
||||
direction TB
|
||||
SQL20["20_stage_events.sql<br/>Normalize & validate"]
|
||||
SQL21["21_build_episodes.sql<br/>Group episodes (30-day gap)"]
|
||||
SQL22["22_build_features.sql<br/>Point-in-time feature windows"]
|
||||
SQL20 --> SQL21 --> SQL22
|
||||
end
|
||||
|
||||
DUCKDB --> MART["inspection_feature_mart.parquet<br/>(episode × features, temporal partitions)"]
|
||||
|
||||
%% Temporal Partitions
|
||||
subgraph Partitions["Temporal Splits"]
|
||||
TRAIN["Train: 2016–2022"]
|
||||
TUNE["Tune: 2023"]
|
||||
CAL["Calibrate: 2024"]
|
||||
TEST["Locked Test: 2025"]
|
||||
DRIFT["Shadow Drift: 2026"]
|
||||
end
|
||||
MART -.- Partitions
|
||||
|
||||
%% Model Training
|
||||
MART -->|"train_baselines.py"| BL["artifacts/baselines/baseline_v1/<br/>Logistic Regression +<br/>Prevalence & Prior-Outcome"]
|
||||
MART -->|"train_tree_model.py"| TREE["artifacts/tree/hist_gradient_boosting_v1/<br/>Histogram Gradient Boosting"]
|
||||
|
||||
%% Model Outputs
|
||||
BL --> ARTIFACTS["Model Artifacts<br/>manifest.json, metrics.json,<br/>model.joblib, calibration_bins.csv"]
|
||||
TREE --> ARTIFACTS
|
||||
|
||||
%% Dashboard Export
|
||||
MART -->|"export_dashboard_data.py<br/>(privacy suppression, aggregation)"| JSON
|
||||
|
||||
subgraph JSON["dashboard/public/data/"]
|
||||
direction TB
|
||||
DM["data_manifest.json"]
|
||||
OV["overview_period_county.json"]
|
||||
CS["cohort_scorecard.json"]
|
||||
ARC["age_risk_curve.json"]
|
||||
CQ["coverage_quality.json"]
|
||||
FC["filter_catalog.json"]
|
||||
MD["model_diagnostics.json"]
|
||||
end
|
||||
|
||||
ARTIFACTS -.->|metrics & manifest| JSON
|
||||
|
||||
%% Dashboard Frontend
|
||||
JSON --> SERVER["server.mjs<br/>Node.js HTTP server"]
|
||||
|
||||
subgraph Dashboard["Browser SPA"]
|
||||
direction TB
|
||||
APP["app.js<br/>Routing & pages"]
|
||||
CHARTS["charts.js<br/>D3 visualizations"]
|
||||
DATA["data.js<br/>Data loading & filters"]
|
||||
APP --- CHARTS
|
||||
APP --- DATA
|
||||
end
|
||||
|
||||
SERVER --> Dashboard
|
||||
|
||||
subgraph Pages["Dashboard Pages"]
|
||||
P1["Overview<br/>KPIs, trends, county map"]
|
||||
P2["Reliability Explorer<br/>Make/model comparison"]
|
||||
P3["Next-Test Estimator<br/>Risk prediction"]
|
||||
P4["Data & Methods<br/>Coverage, limitations"]
|
||||
end
|
||||
|
||||
Dashboard --> Pages
|
||||
|
||||
%% Privacy Controls
|
||||
subgraph Privacy["Privacy Controls"]
|
||||
subgraph SPLITS[Private development chronology]
|
||||
direction LR
|
||||
S1["≥100 inspections per cell"]
|
||||
S2["≥100 distinct vehicles"]
|
||||
S3["≥10 outcome records"]
|
||||
S4["No VINs/plates/ZIPs"]
|
||||
CONTEXT[2010-15<br/>history context]
|
||||
TRAIN[2016-22<br/>train]
|
||||
TUNE[2023<br/>tune]
|
||||
CAL[2024<br/>calibrate]
|
||||
HOLDOUT[2025<br/>one-time holdout]
|
||||
SHADOW[2026 partial<br/>shadow context]
|
||||
end
|
||||
MART -.-> SPLITS
|
||||
|
||||
JSON -.- Privacy
|
||||
MART --> LOGISTIC[Regularized logistic + Platt<br/><b>final prototype model</b>]
|
||||
MART --> TREE[Histogram gradient boosting + Platt<br/><b>benchmark only</b>]
|
||||
|
||||
%% Testing
|
||||
subgraph Tests["Test Suite"]
|
||||
LOGISTIC --> PRIVATE[Private model artifacts<br/>manifests, checksums, metrics]
|
||||
TREE --> PRIVATE
|
||||
HOLDOUT -.-> REPORT[Private/report-only<br/>one-time comparison]
|
||||
PRIVATE -.-> REPORT
|
||||
|
||||
MART --> EXPORT[Aggregate, suppress,<br/>round, validate]
|
||||
PRIVATE -.->|pre-2025 diagnostics only| EXPORT
|
||||
|
||||
subgraph PUBLIC[dashboard/public/data - public boundary]
|
||||
direction TB
|
||||
MANIFEST[data_manifest.json]
|
||||
OVERVIEW[overview_period_county.json]
|
||||
COHORTS[cohort_scorecard.json]
|
||||
AGE[age_risk_curve.json]
|
||||
COVERAGE[coverage_quality.json]
|
||||
FILTERS[filter_catalog.json]
|
||||
DIAGNOSTICS[model_diagnostics.json]
|
||||
HASHES[sha256_manifest.json]
|
||||
end
|
||||
EXPORT --> PUBLIC
|
||||
|
||||
PUBLIC --> CONTRACT[Fail-closed browser validation]
|
||||
CONTRACT --> SITE[Static dashboard<br/>semantic HTML + CSS<br/>vanilla ES modules + inline SVG]
|
||||
|
||||
subgraph VIEWS[Presentation-ready views]
|
||||
direction TB
|
||||
V1[Overview]
|
||||
V2[Sample cohorts]
|
||||
V3[Model & benchmark]
|
||||
V4[Data & methods]
|
||||
end
|
||||
SITE --> VIEWS
|
||||
|
||||
subgraph GUARDRAILS[Publication guardrails]
|
||||
direction TB
|
||||
G1[Private 10,000-vehicle sample<br/>not population estimates]
|
||||
G2[Minimum episode, vehicle,<br/>and binary-class support]
|
||||
G3[No identifiers, raw/operational rows,<br/>credentials, or row-level predictions]
|
||||
G4[No county population rankings<br/>or vehicle-level prediction service]
|
||||
end
|
||||
EXPORT -.-> GUARDRAILS
|
||||
|
||||
SITE --> BOLT[Separate dashboard-only<br/>Bolt project]
|
||||
|
||||
subgraph TESTS[Verification]
|
||||
direction LR
|
||||
PT["Python unittest<br/>(feature mart, models,<br/>export, baselines)"]
|
||||
CT["Node.js contract tests<br/>(dashboard JSON schemas)"]
|
||||
PY[Python pipeline/model/export tests]
|
||||
JS[Node dashboard contract tests]
|
||||
end
|
||||
MART -.-> PY
|
||||
PUBLIC -.-> JS
|
||||
|
||||
MART -.- PT
|
||||
JSON -.- CT
|
||||
classDef private fill:#fff3e0,stroke:#b45309,color:#3b2415
|
||||
classDef model fill:#e0f2fe,stroke:#0369a1,color:#0c4a6e
|
||||
classDef public fill:#ecfdf5,stroke:#047857,color:#064e3b
|
||||
classDef warning fill:#fef2f2,stroke:#b91c1c,color:#7f1d1d
|
||||
|
||||
%% Styling
|
||||
classDef source fill:#e1f5fe,stroke:#0288d1
|
||||
classDef process fill:#fff3e0,stroke:#f57c00
|
||||
classDef storage fill:#e8f5e9,stroke:#388e3c
|
||||
classDef dashboard fill:#fce4ec,stroke:#c62828
|
||||
classDef test fill:#f3e5f5,stroke:#7b1fa2
|
||||
|
||||
class PG,DMV,INSP source
|
||||
class CSV,MART,ARTIFACTS,DUCKDB storage
|
||||
class BL,TREE process
|
||||
class SERVER,Dashboard,Pages,JSON dashboard
|
||||
class Tests test
|
||||
class SOURCE,SAMPLE,STAGE,EPISODES,MART,PRIVATE,REPORT private
|
||||
class LOGISTIC,TREE model
|
||||
class PUBLIC,CONTRACT,SITE,VIEWS,BOLT public
|
||||
class GUARDRAILS warning
|
||||
|
||||
Reference in New Issue
Block a user