diff --git a/dashboard/README.md b/dashboard/README.md index 4bc52bd..62ee7f7 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -57,6 +57,12 @@ contains a denied identifier-like field, the dashboard shows an unavailable state and no estimates. Regenerate assets with the repository's private local pipeline; never hand-edit public JSON to bypass suppression. +Previously published cohort definitions are frozen against cross-release +differencing. New canonical model families may be added only when the combined +family clears every publication threshold; aliases or trims must not be folded +into an existing public cell. Increment `PUBLICATION_POLICY_VERSION` whenever +an approved aggregation policy changes. + The site intentionally contains no vehicle lookup, individual estimator, operational connection, or row-level prediction output. VINs, plates, ZIPs, stations, technician identifiers, raw source JSON, credentials, and operational diff --git a/dashboard/index.html b/dashboard/index.html index 186223a..cd09872 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -126,7 +126,7 @@
-

Availability only—this is not a county outcome ranking. Teal feeds are represented in the sample; gray counties are unavailable, not zero.

+

Availability only—this is not a county outcome ranking. Teal markers show counties represented in the sample; unmarked counties are unavailable, not zero.

@@ -177,7 +177,7 @@ Make or model -

Published grain: prior make and model only. County, age, fuel, program, and period slices are not available for these scorecards.

+

Published grain: prior make and model. Combined model families: Chevrolet Silverado 1500; Dodge Ram 1500; Hyundai Elantra; Nissan Altima; Subaru Outback; and Toyota 4Runner, Corolla, and Tacoma. Exact source-label cohorts: Ford F150; Honda Accord and Civic; and Toyota Camry. County, age, fuel, program, and period slices are not available for these scorecards.

@@ -200,7 +200,7 @@

Descriptive sample comparison

Observed cohort associations

-

Ordering is a viewing aid within this sample, not a reliability ranking or population comparison.

+

The filter note distinguishes combined model families from exact source-label cohorts. Ordering is a viewing aid within this sample, not a reliability ranking or population comparison.

diff --git a/dashboard/js/app.js b/dashboard/js/app.js index 210069c..d43d45f 100644 --- a/dashboard/js/app.js +++ b/dashboard/js/app.js @@ -230,9 +230,16 @@ function updateExplorer() { scorecardLabel(row).toLowerCase().includes(query), ); visibleScorecards = sortedScorecards(matches, sortMode); + const cohortCount = visibleScorecards.length; + let summary = "0 supported sample cohorts."; + if (cohortCount > 0 && cohortCount <= 12) { + summary = `${cohortCount} supported sample cohort${cohortCount === 1 ? "" : "s"}; all shown.`; + } else if (cohortCount > 12) { + summary = `${cohortCount} supported sample cohorts; showing ${Math.min(cohortCount, 18)} cards and 12 chart rows.`; + } setText( "result-summary", - `${visibleScorecards.length} supported sample cohort${visibleScorecards.length === 1 ? "" : "s"}; showing up to 18 cards and 12 chart rows.`, + summary, ); byId("empty-results").hidden = visibleScorecards.length > 0; renderCohortDotPlot(byId("cohort-dot-plot"), visibleScorecards); diff --git a/dashboard/js/charts.js b/dashboard/js/charts.js index 17fe53e..8a180eb 100644 --- a/dashboard/js/charts.js +++ b/dashboard/js/charts.js @@ -205,37 +205,50 @@ export function renderCohortDotPlot(container, rows) { ); } +// Simplified from the Census 2025 1:20m cartographic boundary for Utah +// (GEOID 49). Utah's defining step is on the northeast edge, not both sides. +const UTAH_OUTLINE_PATH = "M44 20H153V67H226V255H44Z"; + +// Census county centers projected into the same Utah map frame. const COUNTY_POINTS = { - cache: [154, 40], - weber: [137, 86], - davis: [126, 108], - salt_lake: [133, 133], - "salt lake": [133, 133], - utah: [140, 170], + cache: [128, 33], + weber: [123, 54], + davis: [111, 65], + "salt lake": [121, 83], + utah: [131, 108], }; +function normalizeCountyName(county) { + return String(county).trim().toLowerCase().replaceAll("_", " ").replace(/\s+/g, " "); +} + +function displayCountyName(county) { + return county.replace(/\b\w/g, (letter) => letter.toUpperCase()); +} + export function renderUtahCoverage(container, coveredCounties) { - const normalized = new Set(coveredCounties.map((county) => county.toLowerCase())); + const normalized = new Set(coveredCounties.map(normalizeCountyName)); const points = [...normalized] .map((county) => ({ county, coordinates: COUNTY_POINTS[county] })) .filter((item) => item.coordinates); + const countyNames = [...normalized].map(displayCountyName); container.innerHTML = ` `; container.setAttribute( "aria-label", - coveredCounties.length - ? `Development-sample feed availability includes ${coveredCounties.join(", ")}. This is not an outcome ranking; other counties are unavailable.` - : "No county feed coverage is available.", + countyNames.length + ? `Outline of Utah showing development-sample feed availability for ${countyNames.join(", ")} County feeds. This is not an outcome ranking; other counties are unavailable.` + : "Outline of Utah. No county feed coverage is available.", ); } diff --git a/dashboard/public/data/cohort_scorecard.json b/dashboard/public/data/cohort_scorecard.json index a09c894..4a5ac0b 100644 --- a/dashboard/public/data/cohort_scorecard.json +++ b/dashboard/public/data/cohort_scorecard.json @@ -1 +1 @@ -{"development_preview":true,"population_estimate_allowed":false,"rows":[{"nonpass_rate":0.116,"prior_make":"FORD","prior_model":"F150","support_rounded":1100},{"nonpass_rate":0.09,"prior_make":"HONDA","prior_model":"ACCORD","support_rounded":400},{"nonpass_rate":0.073,"prior_make":"HONDA","prior_model":"CIVIC","support_rounded":300},{"nonpass_rate":0.086,"prior_make":"TOYOTA","prior_model":"CAMRY","support_rounded":500}],"schema_version":"dashboard_data_v1"} +{"development_preview":true,"population_estimate_allowed":false,"rows":[{"nonpass_rate":0.123,"prior_make":"CHEVROLET","prior_model":"SILVERADO 1500","support_rounded":500},{"nonpass_rate":0.117,"prior_make":"DODGE","prior_model":"RAM 1500","support_rounded":600},{"nonpass_rate":0.116,"prior_make":"FORD","prior_model":"F150","support_rounded":1100},{"nonpass_rate":0.09,"prior_make":"HONDA","prior_model":"ACCORD","support_rounded":400},{"nonpass_rate":0.073,"prior_make":"HONDA","prior_model":"CIVIC","support_rounded":300},{"nonpass_rate":0.094,"prior_make":"HYUNDAI","prior_model":"ELANTRA","support_rounded":500},{"nonpass_rate":0.128,"prior_make":"NISSAN","prior_model":"ALTIMA","support_rounded":500},{"nonpass_rate":0.084,"prior_make":"SUBARU","prior_model":"OUTBACK","support_rounded":600},{"nonpass_rate":0.08,"prior_make":"TOYOTA","prior_model":"4RUNNER","support_rounded":600},{"nonpass_rate":0.086,"prior_make":"TOYOTA","prior_model":"CAMRY","support_rounded":500},{"nonpass_rate":0.082,"prior_make":"TOYOTA","prior_model":"COROLLA","support_rounded":800},{"nonpass_rate":0.08,"prior_make":"TOYOTA","prior_model":"TACOMA","support_rounded":600}],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/data_manifest.json b/dashboard/public/data/data_manifest.json index 44bf720..7473eb9 100644 --- a/dashboard/public/data/data_manifest.json +++ b/dashboard/public/data/data_manifest.json @@ -1 +1 @@ -{"assets":["age_risk_curve.json","cohort_scorecard.json","coverage_quality.json","filter_catalog.json","model_diagnostics.json","overview_period_county.json"],"data_scope":{"first_year":2016,"last_year":2024,"model_names":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"]},"definitions":{"episode_gap_days":30,"locked_test_metrics_published":false,"support_rounding":100,"suppression_min_distinct_nonpass_vehicles":10,"suppression_min_distinct_pass_vehicles":10,"suppression_min_distinct_vehicles":100,"suppression_min_nonpass":10,"suppression_min_pass":10,"suppression_min_support":100,"target":"first-attempt next-episode binary non-pass rate"},"development_preview":true,"model_versions":["baseline_v1","hist_gradient_boosting_v1"],"population_estimate_allowed":false,"release_id":"7c1af8d22d3841b84d2cf4cd5ba7bce6a43578c78bc3d32a25d200b4b1986704","schema_version":"dashboard_data_v1"} +{"assets":["age_risk_curve.json","cohort_scorecard.json","coverage_quality.json","filter_catalog.json","model_diagnostics.json","overview_period_county.json"],"data_scope":{"first_year":2016,"last_year":2024,"model_names":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"]},"definitions":{"episode_gap_days":30,"locked_test_metrics_published":false,"support_rounding":100,"suppression_min_distinct_nonpass_vehicles":10,"suppression_min_distinct_pass_vehicles":10,"suppression_min_distinct_vehicles":100,"suppression_min_nonpass":10,"suppression_min_pass":10,"suppression_min_support":100,"target":"first-attempt next-episode binary non-pass rate"},"development_preview":true,"model_versions":["baseline_v1","hist_gradient_boosting_v1"],"population_estimate_allowed":false,"release_id":"7d207baa6389ff26d6d9fa39da8fd917500bca5f7bdc0ecd090e9ef1a7ca105e","schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/filter_catalog.json b/dashboard/public/data/filter_catalog.json index 83721a2..4f2cd2c 100644 --- a/dashboard/public/data/filter_catalog.json +++ b/dashboard/public/data/filter_catalog.json @@ -1 +1 @@ -{"age_bands":["0-3","4-7","8-11","12-15","16-20","21+"],"development_preview":true,"models":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"],"periods":[{"quarters":[1,2,3,4],"year":2016},{"quarters":[1,2,3,4],"year":2017},{"quarters":[1,2,3,4],"year":2018},{"quarters":[1,2,3,4],"year":2019},{"quarters":[1,2,3,4],"year":2020},{"quarters":[1,2,3,4],"year":2021},{"quarters":[1,2,3,4],"year":2022},{"quarters":[1,2,3,4],"year":2023},{"quarters":[1,2,3,4],"year":2024}],"population_estimate_allowed":false,"prior_make_models":[{"prior_make":"FORD","prior_model":"F150"},{"prior_make":"HONDA","prior_model":"ACCORD"},{"prior_make":"HONDA","prior_model":"CIVIC"},{"prior_make":"TOYOTA","prior_model":"CAMRY"}],"public_counties":["salt_lake","utah","weber"],"schema_version":"dashboard_data_v1"} +{"age_bands":["0-3","4-7","8-11","12-15","16-20","21+"],"development_preview":true,"models":["hist_gradient_boosting_platt","hist_gradient_boosting_raw","logistic_platt","logistic_raw","previous_episode_literal","training_prevalence"],"partitions":["train","tune","calibrate"],"periods":[{"quarters":[1,2,3,4],"year":2016},{"quarters":[1,2,3,4],"year":2017},{"quarters":[1,2,3,4],"year":2018},{"quarters":[1,2,3,4],"year":2019},{"quarters":[1,2,3,4],"year":2020},{"quarters":[1,2,3,4],"year":2021},{"quarters":[1,2,3,4],"year":2022},{"quarters":[1,2,3,4],"year":2023},{"quarters":[1,2,3,4],"year":2024}],"population_estimate_allowed":false,"prior_make_models":[{"prior_make":"CHEVROLET","prior_model":"SILVERADO 1500"},{"prior_make":"DODGE","prior_model":"RAM 1500"},{"prior_make":"FORD","prior_model":"F150"},{"prior_make":"HONDA","prior_model":"ACCORD"},{"prior_make":"HONDA","prior_model":"CIVIC"},{"prior_make":"HYUNDAI","prior_model":"ELANTRA"},{"prior_make":"NISSAN","prior_model":"ALTIMA"},{"prior_make":"SUBARU","prior_model":"OUTBACK"},{"prior_make":"TOYOTA","prior_model":"4RUNNER"},{"prior_make":"TOYOTA","prior_model":"CAMRY"},{"prior_make":"TOYOTA","prior_model":"COROLLA"},{"prior_make":"TOYOTA","prior_model":"TACOMA"}],"public_counties":["salt_lake","utah","weber"],"schema_version":"dashboard_data_v1"} diff --git a/dashboard/public/data/sha256_manifest.json b/dashboard/public/data/sha256_manifest.json index 86b1e1e..72b2e5d 100644 --- a/dashboard/public/data/sha256_manifest.json +++ b/dashboard/public/data/sha256_manifest.json @@ -1 +1 @@ -{"development_preview":true,"files":[{"name":"age_risk_curve.json","sha256":"5445d52365ad3494ba05f097e7f4f5e41fffeecfe50fbb2952d72044f496c925"},{"name":"cohort_scorecard.json","sha256":"2ae83d9d55aa1f2ccae6420c66350fb491405eedfa90374aa3df2782a4aa0bf1"},{"name":"coverage_quality.json","sha256":"35b7dbed50e1b8260aac18c269f5045010ea23649a649f984a4693f3f00a1e83"},{"name":"data_manifest.json","sha256":"b7196b04eb223683c928cfb5375ed618e1230fa03382bd8a889de24c589557a0"},{"name":"filter_catalog.json","sha256":"9e45d428cd38853002ebd0bda089eb46d0832cd1b7b6b05ac953bc8472b01eb5"},{"name":"model_diagnostics.json","sha256":"ebf99eb32436a5eabb7d754cd88109f2cc0981575dd28966757031ac7faafa57"},{"name":"overview_period_county.json","sha256":"fe2a4233563b47fa31fc87859b100983d3bfb574c5a7fbc7241b9cfb4661b038"}],"population_estimate_allowed":false,"schema_version":"dashboard_data_v1"} +{"development_preview":true,"files":[{"name":"age_risk_curve.json","sha256":"5445d52365ad3494ba05f097e7f4f5e41fffeecfe50fbb2952d72044f496c925"},{"name":"cohort_scorecard.json","sha256":"3acd3aa5a32f53e8f0a2baaad6fec8bb0610efaac8151d8ec4a88285faca23fd"},{"name":"coverage_quality.json","sha256":"35b7dbed50e1b8260aac18c269f5045010ea23649a649f984a4693f3f00a1e83"},{"name":"data_manifest.json","sha256":"8ed1593dd223ddd2fbdcd808040ee6a9f37f078011e821a058a1c7a6d94694d2"},{"name":"filter_catalog.json","sha256":"e78e65ae463edf4ea1b5c757e1a002b5fd953beb75c31381f8264e267295f48e"},{"name":"model_diagnostics.json","sha256":"ebf99eb32436a5eabb7d754cd88109f2cc0981575dd28966757031ac7faafa57"},{"name":"overview_period_county.json","sha256":"fe2a4233563b47fa31fc87859b100983d3bfb574c5a7fbc7241b9cfb4661b038"}],"population_estimate_allowed":false,"schema_version":"dashboard_data_v1"} diff --git a/dashboard/styles.css b/dashboard/styles.css index ed7ec2c..b2f1b76 100644 --- a/dashboard/styles.css +++ b/dashboard/styles.css @@ -658,6 +658,22 @@ main { margin: 0 auto 1rem; } +.utah-map .axis-label, +.utah-map .state-label { + fill: var(--ink-650); + font-family: var(--font-sans); +} + +.utah-map .axis-label { + font-size: 11px; +} + +.utah-map .state-label { + font-size: 12px; + font-weight: 700; + letter-spacing: 0.16em; +} + .county-list { display: flex; flex-wrap: wrap; @@ -716,6 +732,10 @@ main { align-items: start; } +.explorer-results { + min-width: 0; +} + .filter-panel { position: sticky; top: 155px; @@ -1522,6 +1542,15 @@ fieldset:disabled .field select { justify-content: flex-start; } + .chart--scorecard { + overflow-x: auto; + overscroll-behavior-inline: contain; + } + + .chart--scorecard svg { + min-width: 720px; + } + .locked-panel { display: block; } diff --git a/dashboard/tests/contract.test.mjs b/dashboard/tests/contract.test.mjs index c36e2d4..5088be5 100644 --- a/dashboard/tests/contract.test.mjs +++ b/dashboard/tests/contract.test.mjs @@ -12,6 +12,7 @@ import { loadDashboardData, validateAssetSet, } from "../js/data.js"; +import { renderUtahCoverage } from "../js/charts.js"; const DASHBOARD_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const PUBLIC_DATA = path.join(DASHBOARD_ROOT, "public", "data"); @@ -84,13 +85,41 @@ test("narrow layouts constrain body content while preserving local nav scrolling assert.match(css, /html\s*{[^}]*max-width:\s*100%;[^}]*overflow-x:\s*clip;/s); assert.match(css, /body\s*{[^}]*max-width:\s*100%;[^}]*overflow-x:\s*clip;/s); assert.match(css, /\.primary-nav__inner\s*{[^}]*overflow-x:\s*auto;/s); + assert.match(css, /\.explorer-results\s*{[^}]*min-width:\s*0;/s); const mobile = css.match(/@media \(max-width: 660px\)\s*{[\s\S]*?(?=@media|$)/)?.[0]; assert.ok(mobile, "mobile breakpoint is present"); assert.match(mobile, /\.page-shell\s*{[^}]*width:\s*auto;[^}]*max-width:\s*calc\(100% - 2rem\);/s); assert.match(mobile, /\.preview-banner\s*{[^}]*justify-content:\s*flex-start;/s); + assert.match(mobile, /\.chart--scorecard\s*{[^}]*overflow-x:\s*auto;/s); + assert.match(mobile, /\.chart--scorecard svg\s*{[^}]*min-width:\s*720px;/s); assert.match(mobile, /overflow-wrap:\s*anywhere;/); }); +test("county coverage renders Utah's northeast step and geographically aligned feeds", () => { + const attributes = new Map(); + const container = { + innerHTML: "", + setAttribute(name, value) { + attributes.set(name, value); + }, + }; + + renderUtahCoverage(container, ["weber", "salt_lake", "utah"]); + + assert.match( + container.innerHTML, + /data-map-feature="utah-outline" d="M44 20H153V67H226V255H44Z"/, + ); + assert.doesNotMatch(container.innerHTML, /stroke-dasharray/); + assert.equal(container.innerHTML.match(/]*>Weber County/); + assert.match(container.innerHTML, /cx="121" cy="83"[^>]*><title>Salt Lake County/); + assert.match(container.innerHTML, /cx="131" cy="108"[^>]*><title>Utah County/); + assert.match(container.innerHTML, />UTAH<\/text>/); + assert.match(attributes.get("aria-label"), /^Outline of Utah showing/); + assert.doesNotMatch(attributes.get("aria-label"), /_/); +}); + test("model view declares the final model, benchmark role, and publication boundary", () => { const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8"); const modelView = viewSource(html, "model"); @@ -120,6 +149,16 @@ test("every result-facing view visibly labels the sample and non-population scop assert.match(html, /not a reliability ranking or population comparison/i); }); +test("cohort view discloses which rows are combined families", () => { + const html = readFileSync(path.join(DASHBOARD_ROOT, "index.html"), "utf8"); + const cohortView = viewSource(html, "cohorts"); + assert.match(cohortView, /Combined model families:/); + assert.match(cohortView, /Silverado 1500; Dodge Ram 1500; Hyundai Elantra; Nissan Altima; Subaru Outback;/); + assert.match(cohortView, /Toyota 4Runner, Corolla, and Tacoma/); + assert.match(cohortView, /Exact source-label cohorts:/); + assert.match(cohortView, /Ford F150; Honda Accord and Civic; and Toyota Camry/); +}); + test("all required generated assets satisfy the browser contract", () => { const validated = validateAssetSet(assetSet()); assert.equal(validated.manifest.schema_version, SCHEMA_VERSION); @@ -139,6 +178,14 @@ test("all required generated assets satisfy the browser contract", () => { ["train", "tune", "calibrate"], ); assert.ok(validated.overview.rows.length > 0); + assert.equal(validated.scorecards.rows.length, 12); + assert.deepEqual( + validated.filters.prior_make_models, + validated.scorecards.rows.map(({ prior_make, prior_model }) => ({ + prior_make, + prior_model, + })), + ); }); test("browser loader verifies every raw asset digest before rendering", async () => { diff --git a/docs/dashboard_spec.md b/docs/dashboard_spec.md index f2c59ec..069494c 100644 --- a/docs/dashboard_spec.md +++ b/docs/dashboard_spec.md @@ -31,6 +31,13 @@ headlined, or described as population county performance. ### Sample cohorts - Search and compare only supported, suppression-cleared make/model cohorts +- Combine documented source abbreviations and trim labels for newly published + model families before applying the unchanged publication thresholds +- Keep earlier public cohort definitions stable across releases so differences + cannot expose a formerly suppressed slice +- Identify Silverado 1500, Ram 1500, Elantra, Altima, Outback, 4Runner, + Corolla, and Tacoma as combined families; identify F150, Accord, Civic, and + Camry as exact source-label cohorts - Show observed sample non-pass rates and rounded support - Default sorting by support rather than risk - Disable filters or adjusted views that the aggregate bundle cannot support diff --git a/docs/demo_script.md b/docs/demo_script.md index 8230a78..02b3e06 100644 --- a/docs/demo_script.md +++ b/docs/demo_script.md @@ -53,7 +53,9 @@ the “observed sample non-pass” label. **Say:** > This view contains only prior make-and-model cohorts that cleared the -> publication thresholds. I can search supported cohorts, but these are observed +> publication thresholds. Selected source abbreviations and trim labels are +> combined for new model families, while earlier published cohorts keep stable +> definitions. I can search supported cohorts, but these are observed > development-sample associations—not reliability grades, rankings, or > recommendations. Unsupported slices are not inferred in the browser, and > suppressed rows are absent rather than hidden. diff --git a/docs/final_report.md b/docs/final_report.md index 1045c89..ccd1359 100644 --- a/docs/final_report.md +++ b/docs/final_report.md @@ -140,7 +140,9 @@ views: 1. **Overview** shows rounded support, sample outcome patterns, vehicle-age patterns, and county feed coverage. -2. **Sample cohorts** shows only supported make/model sample aggregates; it does +2. **Sample cohorts** shows only supported make/model sample aggregates; + selected source abbreviations and trim labels are combined for new model + families while earlier published cohorts keep stable definitions. It does not issue reliability ratings or recommendations. 3. **Model & benchmark** identifies calibrated logistic regression as final and the boosted tree as benchmark-only. It shows pre-2025, 2024 calibration-fit diff --git a/docs/model_card.md b/docs/model_card.md index 1544c9e..6191554 100644 --- a/docs/model_card.md +++ b/docs/model_card.md @@ -132,7 +132,8 @@ population or external-validation results. - Page sampling is not population-representative. - Source, program, time, and geography are entangled. - Reject and abort are heterogeneous non-pass outcomes. -- Make/model aliases and incomplete feed coverage can distort cohorts. +- Residual make/model aliases outside the documented scorecard mappings and + incomplete feed coverage can distort cohorts. - The one-time 2025 holdout is not an external validation dataset. - No protected attributes are modeled, but that does not establish fairness. - Sample calibration does not establish production calibration. diff --git a/scripts/export_dashboard_data.py b/scripts/export_dashboard_data.py index 234ff71..8d1649f 100644 --- a/scripts/export_dashboard_data.py +++ b/scripts/export_dashboard_data.py @@ -26,6 +26,7 @@ import duckdb PROJECT_ROOT = Path(__file__).resolve().parents[1] PUBLIC_DATA_ROOT = (PROJECT_ROOT / "dashboard/public/data").resolve() SCHEMA_VERSION = "dashboard_data_v1" +PUBLICATION_POLICY_VERSION = "dashboard_publication_policy_v1" ALLOWED_PARTITIONS = ("train", "tune", "calibrate") PARTITION_ORDER = {name: index for index, name in enumerate(ALLOWED_PARTITIONS)} LOCKED_PARTITION_ALIASES = ("test", "locked_test", "locked-test") @@ -641,7 +642,183 @@ def _age_risk_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, obje def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, object]]: raw = connection.execute( """ - SELECT last_observed_make, last_observed_model, + WITH legacy_cells(prior_make, prior_model) AS ( + VALUES + ('FORD', 'F150'), + ('HONDA', 'ACCORD'), + ('HONDA', 'CIVIC'), + ('TOYOTA', 'CAMRY') + ), + make_aliases(raw_make, canonical_make) AS ( + VALUES + ('CHEVR', 'CHEVROLET'), + ('HYUND', 'HYUNDAI'), + ('NISSA', 'NISSAN'), + ('SUBAR', 'SUBARU'), + ('TOYOT', 'TOYOTA') + ), + normalized AS ( + SELECT + safe_mart.last_observed_make AS raw_make, + safe_mart.last_observed_model AS raw_model, + coalesce( + make_aliases.canonical_make, + upper(trim(safe_mart.last_observed_make)) + ) AS normalized_make, + upper(trim(safe_mart.last_observed_model)) AS normalized_model, + exact_legacy.prior_make IS NOT NULL AS is_exact_legacy, + display_legacy.prior_make IS NOT NULL AS collides_with_legacy, + safe_mart.vehicle_token, + safe_mart.target_nonpass + FROM safe_mart + LEFT JOIN make_aliases + ON upper(trim(safe_mart.last_observed_make)) = make_aliases.raw_make + LEFT JOIN legacy_cells AS exact_legacy + ON safe_mart.last_observed_make = exact_legacy.prior_make + AND safe_mart.last_observed_model = exact_legacy.prior_model + LEFT JOIN legacy_cells AS display_legacy + ON trim(safe_mart.last_observed_make) = display_legacy.prior_make + AND trim(safe_mart.last_observed_model) = display_legacy.prior_model + WHERE safe_mart.eligible_returning_target + AND safe_mart.target_nonpass IN (0, 1) + AND safe_mart.last_observed_make IS NOT NULL + AND safe_mart.last_observed_model IS NOT NULL + ), + classified AS ( + SELECT + CASE + -- Only the eight families introduced in this release use + -- alias folding. The four legacy cells never enter this + -- branch, so their published counts retain raw-key meaning. + WHEN normalized_make = 'TOYOTA' + AND ( + normalized_model = 'COROLLA' + OR normalized_model LIKE 'COROLLA %' + ) + AND normalized_model != 'COROLLA IM' + AND normalized_model NOT LIKE 'COROLLA IM %' + THEN 'toyota_corolla' + WHEN normalized_make = 'TOYOTA' + AND ( + normalized_model = '4RUNNER' + OR normalized_model LIKE '4RUNNER %' + ) + THEN 'toyota_4runner' + WHEN normalized_make = 'TOYOTA' + AND ( + normalized_model = 'TACOMA' + OR normalized_model LIKE 'TACOMA %' + ) + THEN 'toyota_tacoma' + WHEN normalized_make = 'DODGE' + AND ( + normalized_model = 'RAM 1500' + OR normalized_model LIKE 'RAM 1500 %' + OR normalized_model = 'RAM PICKUP 1500' + OR normalized_model LIKE 'RAM PICKUP 1500 %' + OR normalized_model = 'RAM1500' + OR normalized_model LIKE 'RAM1500 %' + ) + AND normalized_model NOT LIKE '% VAN' + AND normalized_model NOT LIKE '% VAN %' + THEN 'dodge_ram_1500' + WHEN normalized_make = 'CHEVROLET' + AND ( + normalized_model = 'SILVERADO 1500' + OR normalized_model LIKE 'SILVERADO 1500 %' + OR normalized_model = 'SILVERADO 1500HD' + OR normalized_model LIKE 'SILVERADO 1500HD %' + OR normalized_model = 'SILVERADO C1500' + OR normalized_model LIKE 'SILVERADO C1500 %' + OR normalized_model = 'SILVERADO K1500' + OR normalized_model LIKE 'SILVERADO K1500 %' + OR normalized_model = 'C1500 SILVERADO' + OR normalized_model LIKE 'C1500 SILVERADO %' + OR normalized_model = 'K15 SILVERADO' + OR normalized_model LIKE 'K15 SILVERADO %' + OR normalized_model = 'K1500 SILVERADO' + OR normalized_model LIKE 'K1500 SILVERADO %' + ) + THEN 'chevrolet_silverado_1500' + WHEN normalized_make = 'NISSAN' + AND ( + normalized_model = 'ALTIMA' + OR normalized_model LIKE 'ALTIMA %' + ) + THEN 'nissan_altima' + WHEN normalized_make = 'HYUNDAI' + AND ( + normalized_model = 'ELANTRA' + OR normalized_model LIKE 'ELANTRA %' + ) + THEN 'hyundai_elantra' + WHEN normalized_make = 'SUBARU' + AND ( + normalized_model = 'OUTBACK' + OR normalized_model LIKE 'OUTBACK %' + OR normalized_model = 'LEGACY OUTBACK' + OR normalized_model LIKE 'LEGACY OUTBACK %' + ) + THEN 'subaru_outback' + ELSE NULL + END AS new_family, + * + FROM normalized + ), + publication_input AS ( + -- Exact legacy keys are deliberately copied from the raw columns. + SELECT raw_make AS prior_make, + raw_model AS prior_model, + vehicle_token, + target_nonpass + FROM classified + WHERE is_exact_legacy + + UNION ALL + + -- Preserve the original raw-key behavior for unrelated cohorts. + -- A whitespace variant that would render as a legacy key is + -- withheld rather than creating a duplicate public identity. + SELECT raw_make AS prior_make, + raw_model AS prior_model, + vehicle_token, + target_nonpass + FROM classified + WHERE NOT is_exact_legacy + AND NOT collides_with_legacy + AND new_family IS NULL + + UNION ALL + + SELECT + CASE new_family + WHEN 'chevrolet_silverado_1500' THEN 'CHEVROLET' + WHEN 'dodge_ram_1500' THEN 'DODGE' + WHEN 'hyundai_elantra' THEN 'HYUNDAI' + WHEN 'nissan_altima' THEN 'NISSAN' + WHEN 'subaru_outback' THEN 'SUBARU' + WHEN 'toyota_4runner' THEN 'TOYOTA' + WHEN 'toyota_corolla' THEN 'TOYOTA' + WHEN 'toyota_tacoma' THEN 'TOYOTA' + ELSE NULL + END AS prior_make, + CASE new_family + WHEN 'chevrolet_silverado_1500' THEN 'SILVERADO 1500' + WHEN 'dodge_ram_1500' THEN 'RAM 1500' + WHEN 'hyundai_elantra' THEN 'ELANTRA' + WHEN 'nissan_altima' THEN 'ALTIMA' + WHEN 'subaru_outback' THEN 'OUTBACK' + WHEN 'toyota_4runner' THEN '4RUNNER' + WHEN 'toyota_corolla' THEN 'COROLLA' + WHEN 'toyota_tacoma' THEN 'TACOMA' + ELSE NULL + END AS prior_model, + vehicle_token, + target_nonpass + FROM classified + WHERE new_family IS NOT NULL + ) + SELECT prior_make, prior_model, count(*)::BIGINT AS n, sum(target_nonpass)::BIGINT AS nonpass, count(DISTINCT vehicle_token)::BIGINT AS vehicles, @@ -651,11 +828,7 @@ def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, obj count(DISTINCT vehicle_token) FILTER ( WHERE target_nonpass = 1 )::BIGINT AS nonpass_vehicles - FROM safe_mart - WHERE eligible_returning_target - AND target_nonpass IN (0, 1) - AND last_observed_make IS NOT NULL - AND last_observed_model IS NOT NULL + FROM publication_input GROUP BY 1, 2 HAVING count(*) >= 100 AND sum(target_nonpass) >= 10 @@ -667,7 +840,7 @@ def _scorecard_rows(connection: duckdb.DuckDBPyConnection) -> List[Dict[str, obj AND count(DISTINCT vehicle_token) FILTER ( WHERE target_nonpass = 1 ) >= 10 - ORDER BY n DESC, last_observed_make, last_observed_model + ORDER BY n DESC, prior_make, prior_model LIMIT 200 """ ).fetchall() @@ -784,6 +957,7 @@ def _release_id( ) -> str: material = { "schema_version": SCHEMA_VERSION, + "publication_policy_version": PUBLICATION_POLICY_VERSION, "mart_sha256": mart_digest, "models": [ {"model_version": version, "metrics_sha256": metrics_digest} diff --git a/tests/test_dashboard_export.py b/tests/test_dashboard_export.py index 35099d1..027fc16 100644 --- a/tests/test_dashboard_export.py +++ b/tests/test_dashboard_export.py @@ -117,6 +117,213 @@ class DashboardExportTests(unittest.TestCase): } self.assertEqual(first_bytes, second_bytes) + def test_release_id_includes_publication_policy_version(self) -> None: + mart_digest = "a" * 64 + provenance = [("model_v1", "b" * 64)] + release_id = dashboard_export._release_id(mart_digest, provenance) + + with mock.patch.object( + dashboard_export, + "PUBLICATION_POLICY_VERSION", + dashboard_export.PUBLICATION_POLICY_VERSION + "_changed", + ): + changed_release_id = dashboard_export._release_id( + mart_digest, provenance + ) + + self.assertNotEqual(release_id, changed_release_id) + + def test_scorecards_fold_only_new_family_aliases_before_suppression(self) -> None: + connection = duckdb.connect(":memory:") + try: + connection.execute( + """ + CREATE TABLE safe_mart ( + vehicle_token VARCHAR, + eligible_returning_target BOOLEAN, + target_nonpass INTEGER, + last_observed_make VARCHAR, + last_observed_model VARCHAR + ) + """ + ) + rows = [] + + def add_group(make: str, model: str, nonpass: int = 10) -> None: + group_number = len(rows) + for index in range(60): + rows.append( + ( + "group-{}-vehicle-{}".format(group_number, index), + True, + 1 if index < nonpass else 0, + make, + model, + ) + ) + + # Every raw-string group is below the 100-vehicle threshold. Each + # approved pair reaches 120 only after its explicit aliases merge. + canonical_groups = { + ("CHEVROLET", "SILVERADO 1500"): ( + ("CHEVROLET", "SILVERADO 1500 LT"), + ("CHEVR", "K15 SILVERADO"), + ), + ("DODGE", "RAM 1500"): ( + ("DODGE", "RAM 1500 QUAD"), + ("DODGE", "RAM PICKUP 1500"), + ), + ("HYUNDAI", "ELANTRA"): ( + ("HYUNDAI", "ELANTRA GLS"), + ("HYUND", "ELANTRA"), + ), + ("NISSAN", "ALTIMA"): ( + ("NISSAN", "ALTIMA 2.5 S"), + ("NISSA", "ALTIMA"), + ), + ("SUBARU", "OUTBACK"): ( + ("SUBARU", "LEGACY OUTBACK 2.5I AWD"), + ("SUBAR", "OUTBACK"), + ), + ("TOYOTA", "4RUNNER"): ( + ("TOYOTA", "4RUNNER SR5"), + ("TOYOT", "4RUNNER 4WD"), + ), + ("TOYOTA", "COROLLA"): ( + ("TOYOTA", "COROLLA CE LE S"), + ("TOYOT", "COROLLA"), + ), + ("TOYOTA", "TACOMA"): ( + ("TOYOTA", "TACOMA V6"), + ("TOYOT", "TACOMA 4WD"), + ), + } + for variants in canonical_groups.values(): + for make, model in variants: + add_group(make, model) + + # These separately marketed lines share a prefix with a canonical + # family but must remain distinct and suppressed at this support. + for make, model in ( + ("HONDA", "ACCORD CROSS TOUR EXL"), + ("HONDA", "ACCORD CROSSTOUR"), + ("HONDA", "CIVIC CRX SI"), + ("HONDA", "CIVIC DEL SOL SI"), + ("TOYOT", "CAMRY SOLARA"), + ("TOYOTA", "COROLLA IM"), + ("DODGE", "RAM 1500 VAN"), + ("FORD", "F1500"), + ("NISSAN", "ALTIMAX"), + ("TOYOTA", "CAMRYX"), + ): + add_group(make, model, nonpass=30) + + connection.executemany( + "INSERT INTO safe_mart VALUES (?, ?, ?, ?, ?)", rows + ) + scorecards = dashboard_export._scorecard_rows(connection) + finally: + connection.close() + + self.assertEqual( + [(row["prior_make"], row["prior_model"]) for row in scorecards], + sorted(canonical_groups), + ) + self.assertTrue( + all(row["support_rounded"] == 100 for row in scorecards) + ) + self.assertTrue(all(row["nonpass_rate"] == 0.167 for row in scorecards)) + + def test_legacy_scorecard_cells_are_not_broadened_by_aliases(self) -> None: + connection = duckdb.connect(":memory:") + try: + connection.execute( + """ + CREATE TABLE safe_mart ( + vehicle_token VARCHAR, + eligible_returning_target BOOLEAN, + target_nonpass INTEGER, + last_observed_make VARCHAR, + last_observed_model VARCHAR + ) + """ + ) + rows = [] + + def add_group( + make: str, + model: str, + n: int, + nonpass: int, + ) -> None: + group_number = len(rows) + for index in range(n): + rows.append( + ( + "group-{}-vehicle-{}".format(group_number, index), + True, + 1 if index < nonpass else 0, + make, + model, + ) + ) + + legacy_cells = { + ("FORD", "F150"): 12, + ("HONDA", "ACCORD"): 18, + ("HONDA", "CIVIC"): 24, + ("TOYOTA", "CAMRY"): 30, + } + aliases = { + ("FORD", "F150"): ( + ("FORD", "F150 4WD"), + ("ford", "f150"), + ), + ("HONDA", "ACCORD"): ( + ("HONDA", "ACCORD LX"), + ("honda", "accord"), + ), + ("HONDA", "CIVIC"): ( + ("HONDA", "CIVIC EX"), + ("honda", "civic"), + ), + ("TOYOTA", "CAMRY"): ( + ("TOYOTA", "CAMRY LE"), + ("TOYOT", "CAMRY"), + ), + } + for (make, model), nonpass in legacy_cells.items(): + add_group(make, model, 120, nonpass) + + # This raw key renders as the legacy key after safe-category + # trimming. It must not become a second public identity. + add_group(" " + make + " ", " " + model + " ", 120, 60) + + # Each alias is suppressed alone but would pass support if the + # two were folded together, making accidental broadening clear. + for alias_make, alias_model in aliases[(make, model)]: + add_group(alias_make, alias_model, 60, 30) + + connection.executemany( + "INSERT INTO safe_mart VALUES (?, ?, ?, ?, ?)", rows + ) + scorecards = dashboard_export._scorecard_rows(connection) + finally: + connection.close() + + self.assertEqual(len(scorecards), len(legacy_cells)) + by_cell = { + (row["prior_make"], row["prior_model"]): row + for row in scorecards + } + self.assertEqual(set(by_cell), set(legacy_cells)) + for cell, nonpass in legacy_cells.items(): + with self.subTest(cell=cell): + self.assertEqual(by_cell[cell]["support_rounded"], 100) + self.assertEqual( + by_cell[cell]["nonpass_rate"], round(nonpass / 120, 3) + ) + def test_unlocked_model_manifest_is_refused(self) -> None: manifest = self._read_json(self.model_manifest) manifest["locked_test_evaluated"] = True