578 lines
21 KiB
Python
578 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import duckdb
|
|
|
|
from scripts import export_dashboard_data as dashboard_export
|
|
|
|
|
|
class DashboardExportTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temporary.name)
|
|
self.mart = self.root / "private_feature_mart.parquet"
|
|
self.mart_manifest = self.root / "private_feature_mart.parquet.manifest.json"
|
|
self.model_manifest = self.root / "model_manifest.json"
|
|
self.model_metrics = self.root / "model_metrics.json"
|
|
self.public_root = (self.root / "dashboard/public/data").resolve()
|
|
self._write_mart()
|
|
self._write_mart_manifest()
|
|
self._write_model_files()
|
|
|
|
def tearDown(self) -> None:
|
|
self.temporary.cleanup()
|
|
|
|
def test_development_mart_is_refused_without_preview_flag(self) -> None:
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
with self.assertRaises(dashboard_export.ManifestError):
|
|
self._export(development_preview=False)
|
|
self.assertFalse(self.public_root.exists())
|
|
|
|
def test_preview_is_suppressed_deterministic_and_contains_no_locked_data(self) -> None:
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
paths = self._export(development_preview=True)
|
|
|
|
expected_names = set(dashboard_export.PUBLIC_ASSET_NAMES) | {
|
|
dashboard_export.SHA256_MANIFEST_NAME
|
|
}
|
|
self.assertEqual(set(paths), expected_names)
|
|
|
|
first_bytes = {
|
|
name: (self.public_root / name).read_bytes()
|
|
for name in expected_names
|
|
}
|
|
joined = b"\n".join(first_bytes.values()).lower()
|
|
for forbidden in (
|
|
b"lockedmake",
|
|
b"secret_locked",
|
|
b"vehicle_token",
|
|
b"internal_event_id",
|
|
b'"vin"',
|
|
b'"plate"',
|
|
b'"zip"',
|
|
b'"station"',
|
|
b"episode_start",
|
|
b"target_outcome_label_source",
|
|
b'"prediction"',
|
|
):
|
|
self.assertNotIn(forbidden, joined)
|
|
|
|
for name in expected_names:
|
|
payload = json.loads((self.public_root / name).read_text(encoding="utf-8"))
|
|
self.assertIs(payload["development_preview"], True)
|
|
self.assertIs(payload["population_estimate_allowed"], False)
|
|
|
|
scorecards = self._asset("cohort_scorecard.json")["rows"]
|
|
self.assertEqual(
|
|
scorecards,
|
|
[
|
|
{
|
|
"prior_make": "SAFE",
|
|
"prior_model": "SUPPORTED",
|
|
"support_rounded": 400,
|
|
"nonpass_rate": 0.167,
|
|
}
|
|
],
|
|
)
|
|
overview = self._asset("overview_period_county.json")["rows"]
|
|
self.assertEqual(len(overview), 3)
|
|
self.assertEqual(overview[0]["support_rounded"], 800)
|
|
self.assertNotIn("n", overview[0])
|
|
self.assertNotIn("nonpass_count", overview[0])
|
|
|
|
diagnostics = self._asset("model_diagnostics.json")["rows"]
|
|
self.assertEqual(
|
|
{row["partition"] for row in diagnostics},
|
|
{"train", "tune", "calibrate"},
|
|
)
|
|
self.assertEqual({row["model"] for row in diagnostics}, {"test_model"})
|
|
self.assertTrue(all("episodes" not in row for row in diagnostics))
|
|
|
|
data_manifest = self._asset("data_manifest.json")
|
|
self.assertEqual(data_manifest["model_versions"], ["test_model_v1"])
|
|
self.assertRegex(data_manifest["release_id"], r"^[0-9a-f]{64}$")
|
|
|
|
checksum_manifest = self._asset(dashboard_export.SHA256_MANIFEST_NAME)
|
|
for entry in checksum_manifest["files"]:
|
|
content = (self.public_root / entry["name"]).read_bytes()
|
|
self.assertEqual(hashlib.sha256(content).hexdigest(), entry["sha256"])
|
|
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
self._export(development_preview=True, overwrite=True)
|
|
second_bytes = {
|
|
name: (self.public_root / name).read_bytes()
|
|
for name in expected_names
|
|
}
|
|
self.assertEqual(first_bytes, second_bytes)
|
|
|
|
def test_unlocked_model_manifest_is_refused(self) -> None:
|
|
manifest = self._read_json(self.model_manifest)
|
|
manifest["locked_test_evaluated"] = True
|
|
self._write_json(self.model_manifest, manifest)
|
|
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
with self.assertRaises(dashboard_export.ManifestError):
|
|
self._export(development_preview=True)
|
|
self.assertFalse(self.public_root.exists())
|
|
|
|
def test_locked_metric_row_is_refused_even_when_manifest_is_closed(self) -> None:
|
|
metrics = self._read_json(self.model_metrics)
|
|
locked = dict(metrics[0])
|
|
locked["partition"] = "locked_test"
|
|
metrics.append(locked)
|
|
self._write_json(self.model_metrics, metrics)
|
|
self._refresh_metrics_digest()
|
|
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
with self.assertRaises(dashboard_export.ManifestError):
|
|
self._export(development_preview=True)
|
|
self.assertFalse(self.public_root.exists())
|
|
|
|
def test_metrics_digest_mismatch_is_refused(self) -> None:
|
|
metrics = self._read_json(self.model_metrics)
|
|
metrics[0]["average_precision"] = 0.99
|
|
self._write_json(self.model_metrics, metrics)
|
|
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
with self.assertRaises(dashboard_export.ManifestError):
|
|
self._export(development_preview=True)
|
|
self.assertFalse(self.public_root.exists())
|
|
|
|
def test_overwrite_refuses_unexpected_file(self) -> None:
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
self._export(development_preview=True)
|
|
unexpected = self.public_root / "private-notes.txt"
|
|
unexpected.write_text("must not ride along\n", encoding="utf-8")
|
|
with self.assertRaises(dashboard_export.PublicSchemaError):
|
|
self._export(development_preview=True, overwrite=True)
|
|
self.assertTrue(unexpected.exists())
|
|
|
|
def test_overwrite_refuses_directory_in_public_contract(self) -> None:
|
|
self.public_root.mkdir(parents=True)
|
|
(self.public_root / "data_manifest.json").mkdir()
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
with self.assertRaises(dashboard_export.PublicSchemaError):
|
|
self._export(development_preview=True, overwrite=True)
|
|
|
|
def test_pre_2025_locked_partition_aliases_are_refused(self) -> None:
|
|
for partition in dashboard_export.LOCKED_PARTITION_ALIASES:
|
|
with self.subTest(partition=partition):
|
|
self._move_locked_rows_before_boundary(partition)
|
|
with mock.patch.object(
|
|
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
|
):
|
|
with self.assertRaises(dashboard_export.ManifestError):
|
|
self._export(development_preview=True)
|
|
self.assertFalse(self.public_root.exists())
|
|
|
|
def test_public_validator_rejects_sensitive_keys_and_exact_timestamps(self) -> None:
|
|
base = {
|
|
"schema_version": dashboard_export.SCHEMA_VERSION,
|
|
"development_preview": True,
|
|
"population_estimate_allowed": False,
|
|
}
|
|
for unsafe_field in (
|
|
"technician_id",
|
|
"inspection_technician_code",
|
|
"inspector_identifier",
|
|
"database_password",
|
|
"api_secret_value",
|
|
):
|
|
with self.subTest(unsafe_field=unsafe_field):
|
|
with self.assertRaises(dashboard_export.PublicSchemaError):
|
|
dashboard_export._scan_public_value(
|
|
{unsafe_field: "not-public"}
|
|
)
|
|
|
|
unsafe_key = {
|
|
**base,
|
|
"rows": [
|
|
{
|
|
"year": 2022,
|
|
"quarter": 1,
|
|
"public_county": "salt_lake",
|
|
"support_rounded": 100,
|
|
"nonpass_rate": 0.2,
|
|
"vehicle_token": "not-public",
|
|
}
|
|
],
|
|
}
|
|
with self.assertRaises(dashboard_export.PublicSchemaError):
|
|
dashboard_export.validate_public_asset(
|
|
"overview_period_county.json", unsafe_key
|
|
)
|
|
|
|
exact_timestamp = {
|
|
**base,
|
|
"rows": [
|
|
{
|
|
"year": 2022,
|
|
"quarter": 1,
|
|
"public_county": "2022-01-01T12:34:56",
|
|
"support_rounded": 100,
|
|
"nonpass_rate": 0.2,
|
|
}
|
|
],
|
|
}
|
|
with self.assertRaises(dashboard_export.PublicSchemaError):
|
|
dashboard_export.validate_public_asset(
|
|
"overview_period_county.json", exact_timestamp
|
|
)
|
|
|
|
def _export(
|
|
self,
|
|
development_preview: bool,
|
|
overwrite: bool = False,
|
|
) -> dict:
|
|
return dashboard_export.export_dashboard_data(
|
|
mart=self.mart,
|
|
mart_manifest=self.mart_manifest,
|
|
bundles=[
|
|
dashboard_export.ModelBundle(
|
|
manifest=self.model_manifest,
|
|
metrics=self.model_metrics,
|
|
)
|
|
],
|
|
output_dir=self.public_root,
|
|
development_preview=development_preview,
|
|
overwrite=overwrite,
|
|
)
|
|
|
|
def _write_mart(self) -> None:
|
|
connection = duckdb.connect(":memory:")
|
|
try:
|
|
connection.execute(
|
|
"""
|
|
CREATE TABLE mart (
|
|
vehicle_token VARCHAR,
|
|
is_vin_audit BOOLEAN,
|
|
episode_number BIGINT,
|
|
episode_start TIMESTAMP,
|
|
first_outcome VARCHAR,
|
|
target_outcome_label_source VARCHAR,
|
|
target_nonpass INTEGER,
|
|
eligible_returning_target BOOLEAN,
|
|
temporal_partition VARCHAR,
|
|
public_county VARCHAR,
|
|
source_era VARCHAR,
|
|
vehicle_age INTEGER,
|
|
last_observed_make VARCHAR,
|
|
last_observed_model VARCHAR
|
|
)
|
|
"""
|
|
)
|
|
rows = []
|
|
rows.extend(self._group(120, 20, "SAFE", "SUPPORTED"))
|
|
rows.extend(self._group(99, 19, "LOW", "SUPPORT"))
|
|
rows.extend(self._group(120, 9, "LOW", "NONPASS"))
|
|
rows.extend(self._group(120, 111, "LOW", "PASS"))
|
|
rows.extend(
|
|
self._group(
|
|
120,
|
|
60,
|
|
"ENTITY",
|
|
"LOWTOTAL",
|
|
pass_vehicle_count=40,
|
|
nonpass_vehicle_count=40,
|
|
)
|
|
)
|
|
rows.extend(
|
|
self._group(
|
|
120,
|
|
20,
|
|
"ENTITY",
|
|
"LOWNONPASS",
|
|
nonpass_vehicle_count=9,
|
|
)
|
|
)
|
|
rows.extend(
|
|
self._group(
|
|
120,
|
|
100,
|
|
"ENTITY",
|
|
"LOWPASS",
|
|
pass_vehicle_count=9,
|
|
)
|
|
)
|
|
rows.extend(
|
|
self._group(
|
|
120,
|
|
20,
|
|
"SAFE",
|
|
"SUPPORTED",
|
|
temporal_partition="tune",
|
|
)
|
|
)
|
|
rows.extend(
|
|
self._group(
|
|
120,
|
|
20,
|
|
"SAFE",
|
|
"SUPPORTED",
|
|
temporal_partition="calibrate",
|
|
)
|
|
)
|
|
for index in range(120):
|
|
rows.append(
|
|
(
|
|
"secret_vehicle_token",
|
|
False,
|
|
2,
|
|
"2025-01-15 12:34:56",
|
|
"secret_locked",
|
|
"secret_locked_source",
|
|
999,
|
|
True,
|
|
"locked_test",
|
|
"locked_county",
|
|
"locked_source",
|
|
5,
|
|
"LOCKEDMAKE",
|
|
"LOCKEDMODEL",
|
|
)
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO mart VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
rows,
|
|
)
|
|
connection.table("mart").write_parquet(str(self.mart))
|
|
finally:
|
|
connection.close()
|
|
|
|
@staticmethod
|
|
def _group(
|
|
n: int,
|
|
nonpass: int,
|
|
make: str,
|
|
model: str,
|
|
temporal_partition: str = "train",
|
|
pass_vehicle_count: int = None,
|
|
nonpass_vehicle_count: int = None,
|
|
) -> list:
|
|
partition_dates = {
|
|
"train": "2022-01-15 08:00:00",
|
|
"tune": "2023-01-15 08:00:00",
|
|
"calibrate": "2024-01-15 08:00:00",
|
|
}
|
|
pass_count = n - nonpass
|
|
pass_vehicle_count = pass_count if pass_vehicle_count is None else pass_vehicle_count
|
|
nonpass_vehicle_count = (
|
|
nonpass if nonpass_vehicle_count is None else nonpass_vehicle_count
|
|
)
|
|
rows = []
|
|
for index in range(n):
|
|
adverse = index < nonpass
|
|
class_index = index if adverse else index - nonpass
|
|
class_vehicle_count = (
|
|
nonpass_vehicle_count if adverse else pass_vehicle_count
|
|
)
|
|
outcome = "nonpass" if adverse else "pass"
|
|
vehicle_token = "{}_{}_{}_{}_{}".format(
|
|
make,
|
|
model,
|
|
temporal_partition,
|
|
outcome,
|
|
class_index % class_vehicle_count,
|
|
)
|
|
rows.append(
|
|
(
|
|
vehicle_token,
|
|
False,
|
|
2,
|
|
partition_dates[temporal_partition],
|
|
"fail" if adverse else "pass",
|
|
"overall_result",
|
|
1 if adverse else 0,
|
|
True,
|
|
temporal_partition,
|
|
"salt_lake",
|
|
"slc",
|
|
5,
|
|
make,
|
|
model,
|
|
)
|
|
)
|
|
return rows
|
|
|
|
def _write_mart_manifest(self) -> None:
|
|
connection = duckdb.connect(":memory:")
|
|
try:
|
|
columns = [
|
|
{"name": row[0], "duckdb_type": row[1]}
|
|
for row in connection.execute(
|
|
"DESCRIBE SELECT * FROM read_parquet(?)", [str(self.mart)]
|
|
).fetchall()
|
|
]
|
|
finally:
|
|
connection.close()
|
|
self._write_json(
|
|
self.mart_manifest,
|
|
{
|
|
"build_kind": "private_leakage_safe_inspection_feature_mart",
|
|
"classification": "private_pseudonymized_analytical_mart",
|
|
"source_data_kind": "vehicle_history_development_sample",
|
|
"population_estimate_allowed": False,
|
|
"episode_gap_days": 30,
|
|
"columns": columns,
|
|
"parquet_sha256": self._sha256(self.mart),
|
|
},
|
|
)
|
|
|
|
def _write_model_files(self) -> None:
|
|
connection = duckdb.connect(":memory:")
|
|
try:
|
|
private_support = {
|
|
partition: (episodes, nonpass, vehicles)
|
|
for partition, episodes, nonpass, vehicles in connection.execute(
|
|
"""
|
|
SELECT temporal_partition,
|
|
count(*)::BIGINT,
|
|
sum(target_nonpass)::BIGINT,
|
|
count(DISTINCT vehicle_token)::BIGINT
|
|
FROM read_parquet(?)
|
|
WHERE temporal_partition IN ('train', 'tune', 'calibrate')
|
|
AND eligible_returning_target
|
|
AND target_nonpass IN (0, 1)
|
|
AND NOT is_vin_audit
|
|
GROUP BY 1
|
|
""",
|
|
[str(self.mart)],
|
|
).fetchall()
|
|
}
|
|
finally:
|
|
connection.close()
|
|
|
|
rows = []
|
|
for partition, value in (
|
|
("train", 0.25),
|
|
("tune", 0.24),
|
|
("calibrate", 0.23),
|
|
):
|
|
episodes, nonpass, vehicles = private_support[partition]
|
|
rows.append(
|
|
{
|
|
"model": "test_model",
|
|
"partition": partition,
|
|
"cohort": "non_audit",
|
|
"episodes": episodes,
|
|
"nonpass": nonpass,
|
|
"vehicles": vehicles,
|
|
"average_precision": value,
|
|
"brier": 0.1,
|
|
"log_loss": 0.3,
|
|
"roc_auc": 0.7,
|
|
"top_10_capture": 0.2,
|
|
}
|
|
)
|
|
for model, episodes, nonpass in (
|
|
("low_support", 99, 20),
|
|
("low_nonpass", 120, 9),
|
|
("low_pass", 120, 111),
|
|
("low_vehicles", 120, 20),
|
|
):
|
|
rows.append(
|
|
{
|
|
"model": model,
|
|
"partition": "train",
|
|
"cohort": "non_audit",
|
|
"episodes": episodes,
|
|
"nonpass": nonpass,
|
|
"vehicles": 99 if model == "low_vehicles" else episodes,
|
|
"average_precision": 0.2,
|
|
"brier": 0.1,
|
|
"log_loss": 0.3,
|
|
"roc_auc": 0.7,
|
|
"top_10_capture": 0.2,
|
|
}
|
|
)
|
|
self._write_json(self.model_metrics, rows)
|
|
self._write_json(
|
|
self.model_manifest,
|
|
{
|
|
"classification": "private_model_artifact_no_row_predictions",
|
|
"input_sha256": self._sha256(self.mart),
|
|
"metrics_sha256": self._sha256(self.model_metrics),
|
|
"locked_test_evaluated": False,
|
|
"model_version": "test_model_v1",
|
|
},
|
|
)
|
|
|
|
def _refresh_metrics_digest(self) -> None:
|
|
manifest = self._read_json(self.model_manifest)
|
|
manifest["metrics_sha256"] = self._sha256(self.model_metrics)
|
|
self._write_json(self.model_manifest, manifest)
|
|
|
|
def _move_locked_rows_before_boundary(self, partition: str) -> None:
|
|
rewritten = self.root / "rewritten.parquet"
|
|
connection = duckdb.connect(":memory:")
|
|
try:
|
|
connection.execute(
|
|
"CREATE TABLE rewritten AS SELECT * FROM read_parquet(?)",
|
|
[str(self.mart)],
|
|
)
|
|
connection.execute(
|
|
"""
|
|
UPDATE rewritten
|
|
SET temporal_partition = ?,
|
|
episode_start = TIMESTAMP '2024-12-31 12:34:56'
|
|
WHERE last_observed_make = 'LOCKEDMAKE'
|
|
""",
|
|
[partition],
|
|
)
|
|
connection.table("rewritten").write_parquet(str(rewritten))
|
|
finally:
|
|
connection.close()
|
|
rewritten.replace(self.mart)
|
|
|
|
mart_manifest = self._read_json(self.mart_manifest)
|
|
mart_manifest["parquet_sha256"] = self._sha256(self.mart)
|
|
self._write_json(self.mart_manifest, mart_manifest)
|
|
|
|
model_manifest = self._read_json(self.model_manifest)
|
|
model_manifest["input_sha256"] = self._sha256(self.mart)
|
|
self._write_json(self.model_manifest, model_manifest)
|
|
|
|
def _asset(self, name: str) -> object:
|
|
return self._read_json(self.public_root / name)
|
|
|
|
@staticmethod
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
@staticmethod
|
|
def _write_json(path: Path, value: object) -> None:
|
|
path.write_text(
|
|
json.dumps(value, sort_keys=True, allow_nan=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
@staticmethod
|
|
def _read_json(path: Path) -> object:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|