initial code
This commit is contained in:
@@ -0,0 +1,564 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import duckdb
|
||||
|
||||
from scripts import export_dashboard_data as dashboard_export
|
||||
|
||||
|
||||
class DashboardExportTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name)
|
||||
self.mart = self.root / "private_feature_mart.parquet"
|
||||
self.mart_manifest = self.root / "private_feature_mart.parquet.manifest.json"
|
||||
self.model_manifest = self.root / "model_manifest.json"
|
||||
self.model_metrics = self.root / "model_metrics.json"
|
||||
self.public_root = (self.root / "dashboard/public/data").resolve()
|
||||
self._write_mart()
|
||||
self._write_mart_manifest()
|
||||
self._write_model_files()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temporary.cleanup()
|
||||
|
||||
def test_development_mart_is_refused_without_preview_flag(self) -> None:
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=False)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_preview_is_suppressed_deterministic_and_contains_no_locked_data(self) -> None:
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
paths = self._export(development_preview=True)
|
||||
|
||||
expected_names = set(dashboard_export.PUBLIC_ASSET_NAMES) | {
|
||||
dashboard_export.SHA256_MANIFEST_NAME
|
||||
}
|
||||
self.assertEqual(set(paths), expected_names)
|
||||
|
||||
first_bytes = {
|
||||
name: (self.public_root / name).read_bytes()
|
||||
for name in expected_names
|
||||
}
|
||||
joined = b"\n".join(first_bytes.values()).lower()
|
||||
for forbidden in (
|
||||
b"lockedmake",
|
||||
b"secret_locked",
|
||||
b"vehicle_token",
|
||||
b"internal_event_id",
|
||||
b'"vin"',
|
||||
b'"plate"',
|
||||
b'"zip"',
|
||||
b'"station"',
|
||||
b"episode_start",
|
||||
b"target_outcome_label_source",
|
||||
b'"prediction"',
|
||||
):
|
||||
self.assertNotIn(forbidden, joined)
|
||||
|
||||
for name in expected_names:
|
||||
payload = json.loads((self.public_root / name).read_text(encoding="utf-8"))
|
||||
self.assertIs(payload["development_preview"], True)
|
||||
self.assertIs(payload["population_estimate_allowed"], False)
|
||||
|
||||
scorecards = self._asset("cohort_scorecard.json")["rows"]
|
||||
self.assertEqual(
|
||||
scorecards,
|
||||
[
|
||||
{
|
||||
"prior_make": "SAFE",
|
||||
"prior_model": "SUPPORTED",
|
||||
"support_rounded": 400,
|
||||
"nonpass_rate": 0.167,
|
||||
}
|
||||
],
|
||||
)
|
||||
overview = self._asset("overview_period_county.json")["rows"]
|
||||
self.assertEqual(len(overview), 3)
|
||||
self.assertEqual(overview[0]["support_rounded"], 800)
|
||||
self.assertNotIn("n", overview[0])
|
||||
self.assertNotIn("nonpass_count", overview[0])
|
||||
|
||||
diagnostics = self._asset("model_diagnostics.json")["rows"]
|
||||
self.assertEqual(
|
||||
{row["partition"] for row in diagnostics},
|
||||
{"train", "tune", "calibrate"},
|
||||
)
|
||||
self.assertEqual({row["model"] for row in diagnostics}, {"test_model"})
|
||||
self.assertTrue(all("episodes" not in row for row in diagnostics))
|
||||
|
||||
data_manifest = self._asset("data_manifest.json")
|
||||
self.assertEqual(data_manifest["model_versions"], ["test_model_v1"])
|
||||
self.assertRegex(data_manifest["release_id"], r"^[0-9a-f]{64}$")
|
||||
|
||||
checksum_manifest = self._asset(dashboard_export.SHA256_MANIFEST_NAME)
|
||||
for entry in checksum_manifest["files"]:
|
||||
content = (self.public_root / entry["name"]).read_bytes()
|
||||
self.assertEqual(hashlib.sha256(content).hexdigest(), entry["sha256"])
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
self._export(development_preview=True, overwrite=True)
|
||||
second_bytes = {
|
||||
name: (self.public_root / name).read_bytes()
|
||||
for name in expected_names
|
||||
}
|
||||
self.assertEqual(first_bytes, second_bytes)
|
||||
|
||||
def test_unlocked_model_manifest_is_refused(self) -> None:
|
||||
manifest = self._read_json(self.model_manifest)
|
||||
manifest["locked_test_evaluated"] = True
|
||||
self._write_json(self.model_manifest, manifest)
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_locked_metric_row_is_refused_even_when_manifest_is_closed(self) -> None:
|
||||
metrics = self._read_json(self.model_metrics)
|
||||
locked = dict(metrics[0])
|
||||
locked["partition"] = "locked_test"
|
||||
metrics.append(locked)
|
||||
self._write_json(self.model_metrics, metrics)
|
||||
self._refresh_metrics_digest()
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_metrics_digest_mismatch_is_refused(self) -> None:
|
||||
metrics = self._read_json(self.model_metrics)
|
||||
metrics[0]["average_precision"] = 0.99
|
||||
self._write_json(self.model_metrics, metrics)
|
||||
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_overwrite_refuses_unexpected_file(self) -> None:
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
self._export(development_preview=True)
|
||||
unexpected = self.public_root / "private-notes.txt"
|
||||
unexpected.write_text("must not ride along\n", encoding="utf-8")
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
self._export(development_preview=True, overwrite=True)
|
||||
self.assertTrue(unexpected.exists())
|
||||
|
||||
def test_overwrite_refuses_directory_in_public_contract(self) -> None:
|
||||
self.public_root.mkdir(parents=True)
|
||||
(self.public_root / "data_manifest.json").mkdir()
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
self._export(development_preview=True, overwrite=True)
|
||||
|
||||
def test_pre_2025_locked_partition_aliases_are_refused(self) -> None:
|
||||
for partition in dashboard_export.LOCKED_PARTITION_ALIASES:
|
||||
with self.subTest(partition=partition):
|
||||
self._move_locked_rows_before_boundary(partition)
|
||||
with mock.patch.object(
|
||||
dashboard_export, "PUBLIC_DATA_ROOT", self.public_root
|
||||
):
|
||||
with self.assertRaises(dashboard_export.ManifestError):
|
||||
self._export(development_preview=True)
|
||||
self.assertFalse(self.public_root.exists())
|
||||
|
||||
def test_public_validator_rejects_sensitive_keys_and_exact_timestamps(self) -> None:
|
||||
base = {
|
||||
"schema_version": dashboard_export.SCHEMA_VERSION,
|
||||
"development_preview": True,
|
||||
"population_estimate_allowed": False,
|
||||
}
|
||||
unsafe_key = {
|
||||
**base,
|
||||
"rows": [
|
||||
{
|
||||
"year": 2022,
|
||||
"quarter": 1,
|
||||
"public_county": "salt_lake",
|
||||
"support_rounded": 100,
|
||||
"nonpass_rate": 0.2,
|
||||
"vehicle_token": "not-public",
|
||||
}
|
||||
],
|
||||
}
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
dashboard_export.validate_public_asset(
|
||||
"overview_period_county.json", unsafe_key
|
||||
)
|
||||
|
||||
exact_timestamp = {
|
||||
**base,
|
||||
"rows": [
|
||||
{
|
||||
"year": 2022,
|
||||
"quarter": 1,
|
||||
"public_county": "2022-01-01T12:34:56",
|
||||
"support_rounded": 100,
|
||||
"nonpass_rate": 0.2,
|
||||
}
|
||||
],
|
||||
}
|
||||
with self.assertRaises(dashboard_export.PublicSchemaError):
|
||||
dashboard_export.validate_public_asset(
|
||||
"overview_period_county.json", exact_timestamp
|
||||
)
|
||||
|
||||
def _export(
|
||||
self,
|
||||
development_preview: bool,
|
||||
overwrite: bool = False,
|
||||
) -> dict:
|
||||
return dashboard_export.export_dashboard_data(
|
||||
mart=self.mart,
|
||||
mart_manifest=self.mart_manifest,
|
||||
bundles=[
|
||||
dashboard_export.ModelBundle(
|
||||
manifest=self.model_manifest,
|
||||
metrics=self.model_metrics,
|
||||
)
|
||||
],
|
||||
output_dir=self.public_root,
|
||||
development_preview=development_preview,
|
||||
overwrite=overwrite,
|
||||
)
|
||||
|
||||
def _write_mart(self) -> None:
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE mart (
|
||||
vehicle_token VARCHAR,
|
||||
is_vin_audit BOOLEAN,
|
||||
episode_number BIGINT,
|
||||
episode_start TIMESTAMP,
|
||||
first_outcome VARCHAR,
|
||||
target_outcome_label_source VARCHAR,
|
||||
target_nonpass INTEGER,
|
||||
eligible_returning_target BOOLEAN,
|
||||
temporal_partition VARCHAR,
|
||||
public_county VARCHAR,
|
||||
source_era VARCHAR,
|
||||
vehicle_age INTEGER,
|
||||
last_observed_make VARCHAR,
|
||||
last_observed_model VARCHAR
|
||||
)
|
||||
"""
|
||||
)
|
||||
rows = []
|
||||
rows.extend(self._group(120, 20, "SAFE", "SUPPORTED"))
|
||||
rows.extend(self._group(99, 19, "LOW", "SUPPORT"))
|
||||
rows.extend(self._group(120, 9, "LOW", "NONPASS"))
|
||||
rows.extend(self._group(120, 111, "LOW", "PASS"))
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
60,
|
||||
"ENTITY",
|
||||
"LOWTOTAL",
|
||||
pass_vehicle_count=40,
|
||||
nonpass_vehicle_count=40,
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
20,
|
||||
"ENTITY",
|
||||
"LOWNONPASS",
|
||||
nonpass_vehicle_count=9,
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
100,
|
||||
"ENTITY",
|
||||
"LOWPASS",
|
||||
pass_vehicle_count=9,
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
20,
|
||||
"SAFE",
|
||||
"SUPPORTED",
|
||||
temporal_partition="tune",
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
self._group(
|
||||
120,
|
||||
20,
|
||||
"SAFE",
|
||||
"SUPPORTED",
|
||||
temporal_partition="calibrate",
|
||||
)
|
||||
)
|
||||
for index in range(120):
|
||||
rows.append(
|
||||
(
|
||||
"secret_vehicle_token",
|
||||
False,
|
||||
2,
|
||||
"2025-01-15 12:34:56",
|
||||
"secret_locked",
|
||||
"secret_locked_source",
|
||||
999,
|
||||
True,
|
||||
"locked_test",
|
||||
"locked_county",
|
||||
"locked_source",
|
||||
5,
|
||||
"LOCKEDMAKE",
|
||||
"LOCKEDMODEL",
|
||||
)
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO mart VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
connection.table("mart").write_parquet(str(self.mart))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
@staticmethod
|
||||
def _group(
|
||||
n: int,
|
||||
nonpass: int,
|
||||
make: str,
|
||||
model: str,
|
||||
temporal_partition: str = "train",
|
||||
pass_vehicle_count: int = None,
|
||||
nonpass_vehicle_count: int = None,
|
||||
) -> list:
|
||||
partition_dates = {
|
||||
"train": "2022-01-15 08:00:00",
|
||||
"tune": "2023-01-15 08:00:00",
|
||||
"calibrate": "2024-01-15 08:00:00",
|
||||
}
|
||||
pass_count = n - nonpass
|
||||
pass_vehicle_count = pass_count if pass_vehicle_count is None else pass_vehicle_count
|
||||
nonpass_vehicle_count = (
|
||||
nonpass if nonpass_vehicle_count is None else nonpass_vehicle_count
|
||||
)
|
||||
rows = []
|
||||
for index in range(n):
|
||||
adverse = index < nonpass
|
||||
class_index = index if adverse else index - nonpass
|
||||
class_vehicle_count = (
|
||||
nonpass_vehicle_count if adverse else pass_vehicle_count
|
||||
)
|
||||
outcome = "nonpass" if adverse else "pass"
|
||||
vehicle_token = "{}_{}_{}_{}_{}".format(
|
||||
make,
|
||||
model,
|
||||
temporal_partition,
|
||||
outcome,
|
||||
class_index % class_vehicle_count,
|
||||
)
|
||||
rows.append(
|
||||
(
|
||||
vehicle_token,
|
||||
False,
|
||||
2,
|
||||
partition_dates[temporal_partition],
|
||||
"fail" if adverse else "pass",
|
||||
"overall_result",
|
||||
1 if adverse else 0,
|
||||
True,
|
||||
temporal_partition,
|
||||
"salt_lake",
|
||||
"slc",
|
||||
5,
|
||||
make,
|
||||
model,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
def _write_mart_manifest(self) -> None:
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
columns = [
|
||||
{"name": row[0], "duckdb_type": row[1]}
|
||||
for row in connection.execute(
|
||||
"DESCRIBE SELECT * FROM read_parquet(?)", [str(self.mart)]
|
||||
).fetchall()
|
||||
]
|
||||
finally:
|
||||
connection.close()
|
||||
self._write_json(
|
||||
self.mart_manifest,
|
||||
{
|
||||
"build_kind": "private_leakage_safe_inspection_feature_mart",
|
||||
"classification": "private_pseudonymized_analytical_mart",
|
||||
"source_data_kind": "vehicle_history_development_sample",
|
||||
"population_estimate_allowed": False,
|
||||
"episode_gap_days": 30,
|
||||
"columns": columns,
|
||||
"parquet_sha256": self._sha256(self.mart),
|
||||
},
|
||||
)
|
||||
|
||||
def _write_model_files(self) -> None:
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
private_support = {
|
||||
partition: (episodes, nonpass, vehicles)
|
||||
for partition, episodes, nonpass, vehicles in connection.execute(
|
||||
"""
|
||||
SELECT temporal_partition,
|
||||
count(*)::BIGINT,
|
||||
sum(target_nonpass)::BIGINT,
|
||||
count(DISTINCT vehicle_token)::BIGINT
|
||||
FROM read_parquet(?)
|
||||
WHERE temporal_partition IN ('train', 'tune', 'calibrate')
|
||||
AND eligible_returning_target
|
||||
AND target_nonpass IN (0, 1)
|
||||
AND NOT is_vin_audit
|
||||
GROUP BY 1
|
||||
""",
|
||||
[str(self.mart)],
|
||||
).fetchall()
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
rows = []
|
||||
for partition, value in (
|
||||
("train", 0.25),
|
||||
("tune", 0.24),
|
||||
("calibrate", 0.23),
|
||||
):
|
||||
episodes, nonpass, vehicles = private_support[partition]
|
||||
rows.append(
|
||||
{
|
||||
"model": "test_model",
|
||||
"partition": partition,
|
||||
"cohort": "non_audit",
|
||||
"episodes": episodes,
|
||||
"nonpass": nonpass,
|
||||
"vehicles": vehicles,
|
||||
"average_precision": value,
|
||||
"brier": 0.1,
|
||||
"log_loss": 0.3,
|
||||
"roc_auc": 0.7,
|
||||
"top_10_capture": 0.2,
|
||||
}
|
||||
)
|
||||
for model, episodes, nonpass in (
|
||||
("low_support", 99, 20),
|
||||
("low_nonpass", 120, 9),
|
||||
("low_pass", 120, 111),
|
||||
("low_vehicles", 120, 20),
|
||||
):
|
||||
rows.append(
|
||||
{
|
||||
"model": model,
|
||||
"partition": "train",
|
||||
"cohort": "non_audit",
|
||||
"episodes": episodes,
|
||||
"nonpass": nonpass,
|
||||
"vehicles": 99 if model == "low_vehicles" else episodes,
|
||||
"average_precision": 0.2,
|
||||
"brier": 0.1,
|
||||
"log_loss": 0.3,
|
||||
"roc_auc": 0.7,
|
||||
"top_10_capture": 0.2,
|
||||
}
|
||||
)
|
||||
self._write_json(self.model_metrics, rows)
|
||||
self._write_json(
|
||||
self.model_manifest,
|
||||
{
|
||||
"classification": "private_model_artifact_no_row_predictions",
|
||||
"input_sha256": self._sha256(self.mart),
|
||||
"metrics_sha256": self._sha256(self.model_metrics),
|
||||
"locked_test_evaluated": False,
|
||||
"model_version": "test_model_v1",
|
||||
},
|
||||
)
|
||||
|
||||
def _refresh_metrics_digest(self) -> None:
|
||||
manifest = self._read_json(self.model_manifest)
|
||||
manifest["metrics_sha256"] = self._sha256(self.model_metrics)
|
||||
self._write_json(self.model_manifest, manifest)
|
||||
|
||||
def _move_locked_rows_before_boundary(self, partition: str) -> None:
|
||||
rewritten = self.root / "rewritten.parquet"
|
||||
connection = duckdb.connect(":memory:")
|
||||
try:
|
||||
connection.execute(
|
||||
"CREATE TABLE rewritten AS SELECT * FROM read_parquet(?)",
|
||||
[str(self.mart)],
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE rewritten
|
||||
SET temporal_partition = ?,
|
||||
episode_start = TIMESTAMP '2024-12-31 12:34:56'
|
||||
WHERE last_observed_make = 'LOCKEDMAKE'
|
||||
""",
|
||||
[partition],
|
||||
)
|
||||
connection.table("rewritten").write_parquet(str(rewritten))
|
||||
finally:
|
||||
connection.close()
|
||||
rewritten.replace(self.mart)
|
||||
|
||||
mart_manifest = self._read_json(self.mart_manifest)
|
||||
mart_manifest["parquet_sha256"] = self._sha256(self.mart)
|
||||
self._write_json(self.mart_manifest, mart_manifest)
|
||||
|
||||
model_manifest = self._read_json(self.model_manifest)
|
||||
model_manifest["input_sha256"] = self._sha256(self.mart)
|
||||
self._write_json(self.model_manifest, model_manifest)
|
||||
|
||||
def _asset(self, name: str) -> object:
|
||||
return self._read_json(self.public_root / name)
|
||||
|
||||
@staticmethod
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, sort_keys=True, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _read_json(path: Path) -> object:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user