116 lines
3.5 KiB
Plaintext
116 lines
3.5 KiB
Plaintext
graph TD
|
||
%% Data Sources
|
||
PG[(PostgreSQL<br/>countydata)]
|
||
DMV[("DMV Data<br/>~18M records<br/>2011-2024")]
|
||
INSP[("Inspection Data<br/>~19.3M records<br/>2010-2026")]
|
||
|
||
PG --- DMV
|
||
PG --- INSP
|
||
|
||
%% Extraction
|
||
PG -->|"export_history_sample.py<br/>(VIN hashing, page sampling)"| CSV["history_sample_10000.csv.gz<br/>+ manifest"]
|
||
|
||
%% Feature Mart Build
|
||
CSV -->|"build_feature_mart.py"| DUCKDB
|
||
|
||
subgraph DuckDB["DuckDB Local Warehouse"]
|
||
direction TB
|
||
SQL20["20_stage_events.sql<br/>Normalize & validate"]
|
||
SQL21["21_build_episodes.sql<br/>Group episodes (30-day gap)"]
|
||
SQL22["22_build_features.sql<br/>Point-in-time feature windows"]
|
||
SQL20 --> SQL21 --> SQL22
|
||
end
|
||
|
||
DUCKDB --> MART["inspection_feature_mart.parquet<br/>(episode × features, temporal partitions)"]
|
||
|
||
%% Temporal Partitions
|
||
subgraph Partitions["Temporal Splits"]
|
||
TRAIN["Train: 2016–2022"]
|
||
TUNE["Tune: 2023"]
|
||
CAL["Calibrate: 2024"]
|
||
TEST["Locked Test: 2025"]
|
||
DRIFT["Shadow Drift: 2026"]
|
||
end
|
||
MART -.- Partitions
|
||
|
||
%% Model Training
|
||
MART -->|"train_baselines.py"| BL["artifacts/baselines/baseline_v1/<br/>Logistic Regression +<br/>Prevalence & Prior-Outcome"]
|
||
MART -->|"train_tree_model.py"| TREE["artifacts/tree/hist_gradient_boosting_v1/<br/>Histogram Gradient Boosting"]
|
||
|
||
%% Model Outputs
|
||
BL --> ARTIFACTS["Model Artifacts<br/>manifest.json, metrics.json,<br/>model.joblib, calibration_bins.csv"]
|
||
TREE --> ARTIFACTS
|
||
|
||
%% Dashboard Export
|
||
MART -->|"export_dashboard_data.py<br/>(privacy suppression, aggregation)"| JSON
|
||
|
||
subgraph JSON["dashboard/public/data/"]
|
||
direction TB
|
||
DM["data_manifest.json"]
|
||
OV["overview_period_county.json"]
|
||
CS["cohort_scorecard.json"]
|
||
ARC["age_risk_curve.json"]
|
||
CQ["coverage_quality.json"]
|
||
FC["filter_catalog.json"]
|
||
MD["model_diagnostics.json"]
|
||
end
|
||
|
||
ARTIFACTS -.->|metrics & manifest| JSON
|
||
|
||
%% Dashboard Frontend
|
||
JSON --> SERVER["server.mjs<br/>Node.js HTTP server"]
|
||
|
||
subgraph Dashboard["Browser SPA"]
|
||
direction TB
|
||
APP["app.js<br/>Routing & pages"]
|
||
CHARTS["charts.js<br/>D3 visualizations"]
|
||
DATA["data.js<br/>Data loading & filters"]
|
||
APP --- CHARTS
|
||
APP --- DATA
|
||
end
|
||
|
||
SERVER --> Dashboard
|
||
|
||
subgraph Pages["Dashboard Pages"]
|
||
P1["Overview<br/>KPIs, trends, county map"]
|
||
P2["Reliability Explorer<br/>Make/model comparison"]
|
||
P3["Next-Test Estimator<br/>Risk prediction"]
|
||
P4["Data & Methods<br/>Coverage, limitations"]
|
||
end
|
||
|
||
Dashboard --> Pages
|
||
|
||
%% Privacy Controls
|
||
subgraph Privacy["Privacy Controls"]
|
||
direction LR
|
||
S1["≥100 inspections per cell"]
|
||
S2["≥100 distinct vehicles"]
|
||
S3["≥10 outcome records"]
|
||
S4["No VINs/plates/ZIPs"]
|
||
end
|
||
|
||
JSON -.- Privacy
|
||
|
||
%% Testing
|
||
subgraph Tests["Test Suite"]
|
||
direction LR
|
||
PT["Python unittest<br/>(feature mart, models,<br/>export, baselines)"]
|
||
CT["Node.js contract tests<br/>(dashboard JSON schemas)"]
|
||
end
|
||
|
||
MART -.- PT
|
||
JSON -.- CT
|
||
|
||
%% Styling
|
||
classDef source fill:#e1f5fe,stroke:#0288d1
|
||
classDef process fill:#fff3e0,stroke:#f57c00
|
||
classDef storage fill:#e8f5e9,stroke:#388e3c
|
||
classDef dashboard fill:#fce4ec,stroke:#c62828
|
||
classDef test fill:#f3e5f5,stroke:#7b1fa2
|
||
|
||
class PG,DMV,INSP source
|
||
class CSV,MART,ARTIFACTS,DUCKDB storage
|
||
class BL,TREE process
|
||
class SERVER,Dashboard,Pages,JSON dashboard
|
||
class Tests test
|