diagram of the entire architecture

2026-07-17 15:33:54 -06:00 · 2026-07-17 15:33:54 -06:00 · 4a40e6c765
commit 4a40e6c765
parent 8561f131b0
1 changed files with 115 additions and 0 deletions
--- a/docs/architecture.mmd
+++ b/docs/architecture.mmd
@ -0,0 +1,115 @@
+graph TD
+    %% Data Sources
+    PG[(PostgreSQL<br/>countydata)]
+    DMV[("DMV Data<br/>~18M records<br/>2011-2024")]
+    INSP[("Inspection Data<br/>~19.3M records<br/>2010-2026")]
+
+    PG --- DMV
+    PG --- INSP
+
+    %% Extraction
+    PG -->|"export_history_sample.py<br/>(VIN hashing, page sampling)"| CSV["history_sample_10000.csv.gz<br/>+ manifest"]
+
+    %% Feature Mart Build
+    CSV -->|"build_feature_mart.py"| DUCKDB
+
+    subgraph DuckDB["DuckDB Local Warehouse"]
+        direction TB
+        SQL20["20_stage_events.sql<br/>Normalize & validate"]
+        SQL21["21_build_episodes.sql<br/>Group episodes (30-day gap)"]
+        SQL22["22_build_features.sql<br/>Point-in-time feature windows"]
+        SQL20 --> SQL21 --> SQL22
+    end
+
+    DUCKDB --> MART["inspection_feature_mart.parquet<br/>(episode × features, temporal partitions)"]
+
+    %% Temporal Partitions
+    subgraph Partitions["Temporal Splits"]
+        TRAIN["Train: 2016–2022"]
+        TUNE["Tune: 2023"]
+        CAL["Calibrate: 2024"]
+        TEST["Locked Test: 2025"]
+        DRIFT["Shadow Drift: 2026"]
+    end
+    MART -.- Partitions
+
+    %% Model Training
+    MART -->|"train_baselines.py"| BL["artifacts/baselines/baseline_v1/<br/>Logistic Regression +<br/>Prevalence & Prior-Outcome"]
+    MART -->|"train_tree_model.py"| TREE["artifacts/tree/hist_gradient_boosting_v1/<br/>Histogram Gradient Boosting"]
+
+    %% Model Outputs
+    BL --> ARTIFACTS["Model Artifacts<br/>manifest.json, metrics.json,<br/>model.joblib, calibration_bins.csv"]
+    TREE --> ARTIFACTS
+
+    %% Dashboard Export
+    MART -->|"export_dashboard_data.py<br/>(privacy suppression, aggregation)"| JSON
+
+    subgraph JSON["dashboard/public/data/"]
+        direction TB
+        DM["data_manifest.json"]
+        OV["overview_period_county.json"]
+        CS["cohort_scorecard.json"]
+        ARC["age_risk_curve.json"]
+        CQ["coverage_quality.json"]
+        FC["filter_catalog.json"]
+        MD["model_diagnostics.json"]
+    end
+
+    ARTIFACTS -.->|metrics & manifest| JSON
+
+    %% Dashboard Frontend
+    JSON --> SERVER["server.mjs<br/>Node.js HTTP server"]
+
+    subgraph Dashboard["Browser SPA"]
+        direction TB
+        APP["app.js<br/>Routing & pages"]
+        CHARTS["charts.js<br/>D3 visualizations"]
+        DATA["data.js<br/>Data loading & filters"]
+        APP --- CHARTS
+        APP --- DATA
+    end
+
+    SERVER --> Dashboard
+
+    subgraph Pages["Dashboard Pages"]
+        P1["Overview<br/>KPIs, trends, county map"]
+        P2["Reliability Explorer<br/>Make/model comparison"]
+        P3["Next-Test Estimator<br/>Risk prediction"]
+        P4["Data & Methods<br/>Coverage, limitations"]
+    end
+
+    Dashboard --> Pages
+
+    %% Privacy Controls
+    subgraph Privacy["Privacy Controls"]
+        direction LR
+        S1["≥100 inspections per cell"]
+        S2["≥100 distinct vehicles"]
+        S3["≥10 outcome records"]
+        S4["No VINs/plates/ZIPs"]
+    end
+
+    JSON -.- Privacy
+
+    %% Testing
+    subgraph Tests["Test Suite"]
+        direction LR
+        PT["Python unittest<br/>(feature mart, models,<br/>export, baselines)"]
+        CT["Node.js contract tests<br/>(dashboard JSON schemas)"]
+    end
+
+    MART -.- PT
+    JSON -.- CT
+
+    %% Styling
+    classDef source fill:#e1f5fe,stroke:#0288d1
+    classDef process fill:#fff3e0,stroke:#f57c00
+    classDef storage fill:#e8f5e9,stroke:#388e3c
+    classDef dashboard fill:#fce4ec,stroke:#c62828
+    classDef test fill:#f3e5f5,stroke:#7b1fa2
+
+    class PG,DMV,INSP source
+    class CSV,MART,ARTIFACTS,DUCKDB storage
+    class BL,TREE process
+    class SERVER,Dashboard,Pages,JSON dashboard
+    class Tests test