Centralized_I-M_Database/SummerProject2026
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

various edits

Browse Source
This commit is contained in:
kevinbell
2026-07-21 16:43:22 -06:00
parent 88161a6f16
commit c1e453d962
16 changed files with 526 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/export_dashboard_data.py
+181 -7
View File
@@ -26,6 +26,7 @@ import duckdb
PROJECT_ROOT = Path(__file__).resolve().parents[1]
PUBLIC_DATA_ROOT = (PROJECT_ROOT / "dashboard/public/data").resolve()
SCHEMA_VERSION = "dashboard_data_v1"
PUBLICATION_POLICY_VERSION = "dashboard_publication_policy_v1"
ALLOWED_PARTITIONS = ("train", "tune", "calibrate")
PARTITION_ORDER = {name: index for index, name in enumerate(ALLOWED_PARTITIONS)}
LOCKED_PARTITION_ALIASES = ("test", "locked_test", "locked-test")
@@ -641,7 +642,183 @@ def _age_risk_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, obje
def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, object]]:
raw = connection.execute(
"""
SELECT last_observed_make, last_observed_model,
WITH legacy_cells(prior_make, prior_model) AS (
VALUES
('FORD', 'F150'),
('HONDA', 'ACCORD'),
('HONDA', 'CIVIC'),
('TOYOTA', 'CAMRY')
),
make_aliases(raw_make, canonical_make) AS (
VALUES
('CHEVR', 'CHEVROLET'),
('HYUND', 'HYUNDAI'),
('NISSA', 'NISSAN'),
('SUBAR', 'SUBARU'),
('TOYOT', 'TOYOTA')
),
normalized AS (
SELECT
safe_mart.last_observed_make AS raw_make,
safe_mart.last_observed_model AS raw_model,
coalesce(
make_aliases.canonical_make,
upper(trim(safe_mart.last_observed_make))
) AS normalized_make,
upper(trim(safe_mart.last_observed_model)) AS normalized_model,
exact_legacy.prior_make IS NOT NULL AS is_exact_legacy,
display_legacy.prior_make IS NOT NULL AS collides_with_legacy,
safe_mart.vehicle_token,
safe_mart.target_nonpass
FROM safe_mart
LEFT JOIN make_aliases
ON upper(trim(safe_mart.last_observed_make)) = make_aliases.raw_make
LEFT JOIN legacy_cells AS exact_legacy
ON safe_mart.last_observed_make = exact_legacy.prior_make
AND safe_mart.last_observed_model = exact_legacy.prior_model
LEFT JOIN legacy_cells AS display_legacy
ON trim(safe_mart.last_observed_make) = display_legacy.prior_make
AND trim(safe_mart.last_observed_model) = display_legacy.prior_model
WHERE safe_mart.eligible_returning_target
AND safe_mart.target_nonpass IN (0, 1)
AND safe_mart.last_observed_make IS NOT NULL
AND safe_mart.last_observed_model IS NOT NULL
),
classified AS (
SELECT
CASE
-- Only the eight families introduced in this release use
-- alias folding. The four legacy cells never enter this
-- branch, so their published counts retain raw-key meaning.
WHEN normalized_make = 'TOYOTA'
AND (
normalized_model = 'COROLLA'
OR normalized_model LIKE 'COROLLA %'
)
AND normalized_model != 'COROLLA IM'
AND normalized_model NOT LIKE 'COROLLA IM %'
THEN 'toyota_corolla'
WHEN normalized_make = 'TOYOTA'
AND (
normalized_model = '4RUNNER'
OR normalized_model LIKE '4RUNNER %'
)
THEN 'toyota_4runner'
WHEN normalized_make = 'TOYOTA'
AND (
normalized_model = 'TACOMA'
OR normalized_model LIKE 'TACOMA %'
)
THEN 'toyota_tacoma'
WHEN normalized_make = 'DODGE'
AND (
normalized_model = 'RAM 1500'
OR normalized_model LIKE 'RAM 1500 %'
OR normalized_model = 'RAM PICKUP 1500'
OR normalized_model LIKE 'RAM PICKUP 1500 %'
OR normalized_model = 'RAM1500'
OR normalized_model LIKE 'RAM1500 %'
)
AND normalized_model NOT LIKE '% VAN'
AND normalized_model NOT LIKE '% VAN %'
THEN 'dodge_ram_1500'
WHEN normalized_make = 'CHEVROLET'
AND (
normalized_model = 'SILVERADO 1500'
OR normalized_model LIKE 'SILVERADO 1500 %'
OR normalized_model = 'SILVERADO 1500HD'
OR normalized_model LIKE 'SILVERADO 1500HD %'
OR normalized_model = 'SILVERADO C1500'
OR normalized_model LIKE 'SILVERADO C1500 %'
OR normalized_model = 'SILVERADO K1500'
OR normalized_model LIKE 'SILVERADO K1500 %'
OR normalized_model = 'C1500 SILVERADO'
OR normalized_model LIKE 'C1500 SILVERADO %'
OR normalized_model = 'K15 SILVERADO'
OR normalized_model LIKE 'K15 SILVERADO %'
OR normalized_model = 'K1500 SILVERADO'
OR normalized_model LIKE 'K1500 SILVERADO %'
)
THEN 'chevrolet_silverado_1500'
WHEN normalized_make = 'NISSAN'
AND (
normalized_model = 'ALTIMA'
OR normalized_model LIKE 'ALTIMA %'
)
THEN 'nissan_altima'
WHEN normalized_make = 'HYUNDAI'
AND (
normalized_model = 'ELANTRA'
OR normalized_model LIKE 'ELANTRA %'
)
THEN 'hyundai_elantra'
WHEN normalized_make = 'SUBARU'
AND (
normalized_model = 'OUTBACK'
OR normalized_model LIKE 'OUTBACK %'
OR normalized_model = 'LEGACY OUTBACK'
OR normalized_model LIKE 'LEGACY OUTBACK %'
)
THEN 'subaru_outback'
ELSE NULL
END AS new_family,
*
FROM normalized
),
publication_input AS (
-- Exact legacy keys are deliberately copied from the raw columns.
SELECT raw_make AS prior_make,
raw_model AS prior_model,
vehicle_token,
target_nonpass
FROM classified
WHERE is_exact_legacy
UNION ALL
-- Preserve the original raw-key behavior for unrelated cohorts.
-- A whitespace variant that would render as a legacy key is
-- withheld rather than creating a duplicate public identity.
SELECT raw_make AS prior_make,
raw_model AS prior_model,
vehicle_token,
target_nonpass
FROM classified
WHERE NOT is_exact_legacy
AND NOT collides_with_legacy
AND new_family IS NULL
UNION ALL
SELECT
CASE new_family
WHEN 'chevrolet_silverado_1500' THEN 'CHEVROLET'
WHEN 'dodge_ram_1500' THEN 'DODGE'
WHEN 'hyundai_elantra' THEN 'HYUNDAI'
WHEN 'nissan_altima' THEN 'NISSAN'
WHEN 'subaru_outback' THEN 'SUBARU'
WHEN 'toyota_4runner' THEN 'TOYOTA'
WHEN 'toyota_corolla' THEN 'TOYOTA'
WHEN 'toyota_tacoma' THEN 'TOYOTA'
ELSE NULL
END AS prior_make,
CASE new_family
WHEN 'chevrolet_silverado_1500' THEN 'SILVERADO 1500'
WHEN 'dodge_ram_1500' THEN 'RAM 1500'
WHEN 'hyundai_elantra' THEN 'ELANTRA'
WHEN 'nissan_altima' THEN 'ALTIMA'
WHEN 'subaru_outback' THEN 'OUTBACK'
WHEN 'toyota_4runner' THEN '4RUNNER'
WHEN 'toyota_corolla' THEN 'COROLLA'
WHEN 'toyota_tacoma' THEN 'TACOMA'
ELSE NULL
END AS prior_model,
vehicle_token,
target_nonpass
FROM classified
WHERE new_family IS NOT NULL
)
SELECT prior_make, prior_model,
count(*)::BIGINT AS n,
sum(target_nonpass)::BIGINT AS nonpass,
count(DISTINCT vehicle_token)::BIGINT AS vehicles,
@@ -651,11 +828,7 @@ def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, obj
count(DISTINCT vehicle_token) FILTER (
WHERE target_nonpass = 1
)::BIGINT AS nonpass_vehicles
FROM safe_mart
WHERE eligible_returning_target
AND target_nonpass IN (0, 1)
AND last_observed_make IS NOT NULL
AND last_observed_model IS NOT NULL
FROM publication_input
GROUP BY 1, 2
HAVING count(*) >= 100
AND sum(target_nonpass) >= 10
@@ -667,7 +840,7 @@ def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, obj
AND count(DISTINCT vehicle_token) FILTER (
WHERE target_nonpass = 1
) >= 10
ORDER BY n DESC, last_observed_make, last_observed_model
ORDER BY n DESC, prior_make, prior_model
LIMIT 200
"""
).fetchall()
@@ -784,6 +957,7 @@ def _release_id(
) -> str:
material = {
"schema_version": SCHEMA_VERSION,
"publication_policy_version": PUBLICATION_POLICY_VERSION,
"mart_sha256": mart_digest,
"models": [
{"model_version": version, "metrics_sha256": metrics_digest}