initial code
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
-- Run the whole file when connected to the countydata database.
|
||||
-- The explicit read-only transaction protects against accidental writes.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
SELECT
|
||||
current_database() AS database_name,
|
||||
current_user AS database_user,
|
||||
current_setting('transaction_read_only') AS transaction_read_only,
|
||||
version() AS postgres_version;
|
||||
|
||||
SELECT
|
||||
ssl,
|
||||
version AS tls_version,
|
||||
cipher,
|
||||
bits
|
||||
FROM pg_stat_ssl
|
||||
WHERE pid = pg_backend_pid();
|
||||
|
||||
SELECT
|
||||
n.nspname AS schema_name,
|
||||
c.relname AS relation_name,
|
||||
CASE c.relkind
|
||||
WHEN 'r' THEN 'table'
|
||||
WHEN 'p' THEN 'partitioned table'
|
||||
WHEN 'v' THEN 'view'
|
||||
WHEN 'm' THEN 'materialized view'
|
||||
WHEN 'f' THEN 'foreign table'
|
||||
ELSE c.relkind::text
|
||||
END AS relation_type,
|
||||
COALESCE(s.n_live_tup, c.reltuples)::bigint AS estimated_rows
|
||||
FROM pg_class AS c
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid
|
||||
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_toast%'
|
||||
ORDER BY n.nspname, c.relname;
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,53 @@
|
||||
-- Aggregate-only starter queries. Do not select VIN, plate, email, session,
|
||||
-- upload-log, or raw JSON fields into a public-facing application.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
SELECT
|
||||
count(*) AS registration_records,
|
||||
min(registration_date) AS earliest_registration,
|
||||
max(registration_date) AS latest_registration
|
||||
FROM data.dmv_tax;
|
||||
|
||||
SELECT
|
||||
extract(year FROM registration_date)::integer AS registration_year,
|
||||
count(*) AS records
|
||||
FROM data.dmv_tax
|
||||
GROUP BY 1
|
||||
ORDER BY 1;
|
||||
|
||||
SELECT
|
||||
upper(trim(county)) AS county,
|
||||
count(*) AS records
|
||||
FROM data.dmv_tax
|
||||
GROUP BY 1
|
||||
ORDER BY 2 DESC;
|
||||
|
||||
SELECT
|
||||
upper(replace(trim(fuel_type), '-', ' ')) AS canonical_fuel_type,
|
||||
count(*) AS records
|
||||
FROM data.dmv_tax_vehicle
|
||||
GROUP BY 1
|
||||
ORDER BY 2 DESC;
|
||||
|
||||
SELECT
|
||||
count(*) AS inspection_records,
|
||||
min(test_start) AS earliest_test,
|
||||
max(test_start) AS latest_test
|
||||
FROM data.inspection_search;
|
||||
|
||||
SELECT
|
||||
extract(year FROM test_start)::integer AS test_year,
|
||||
count(*) AS records
|
||||
FROM data.inspection_search
|
||||
GROUP BY 1
|
||||
ORDER BY 1;
|
||||
|
||||
SELECT
|
||||
lower(trim(county)) AS county_source,
|
||||
coalesce(nullif(upper(trim(overall_result)), ''), '<BLANK/NULL>') AS result,
|
||||
count(*) AS records
|
||||
FROM data.inspection_search
|
||||
GROUP BY 1, 2
|
||||
ORDER BY 1, 3 DESC;
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,236 @@
|
||||
-- Aggregate-only feasibility profile for the selected project (label contract v3).
|
||||
--
|
||||
-- This intentionally samples candidate VINs, keeps their complete histories,
|
||||
-- and returns only aggregates. The sample is suitable for cohort development,
|
||||
-- not for final population estimates. Run the whole file on countydata.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
WITH candidate_vins AS MATERIALIZED (
|
||||
SELECT DISTINCT vin
|
||||
FROM data.inspection_search TABLESAMPLE SYSTEM (0.25) REPEATABLE (20260715)
|
||||
WHERE vin IS NOT NULL
|
||||
AND length(btrim(vin)) = 17
|
||||
),
|
||||
sampled_vins AS MATERIALIZED (
|
||||
SELECT vin
|
||||
FROM candidate_vins
|
||||
ORDER BY md5(vin)
|
||||
LIMIT 2000
|
||||
),
|
||||
base AS MATERIALIZED (
|
||||
SELECT
|
||||
s.vin,
|
||||
s.id,
|
||||
s.test_start,
|
||||
lower(btrim(s.county)) AS source_era,
|
||||
CASE
|
||||
WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake'
|
||||
ELSE lower(btrim(s.county))
|
||||
END AS public_county,
|
||||
CASE upper(btrim(s.overall_result))
|
||||
WHEN 'PASS' THEN 'pass'
|
||||
WHEN 'P' THEN 'pass'
|
||||
WHEN 'FAIL' THEN 'fail'
|
||||
WHEN 'F' THEN 'fail'
|
||||
WHEN 'REJECT' THEN 'reject'
|
||||
WHEN 'ABORT' THEN 'abort'
|
||||
ELSE CASE
|
||||
WHEN lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD' THEN
|
||||
CASE upper(btrim(s.obd_result))
|
||||
WHEN 'PASS' THEN 'pass'
|
||||
WHEN 'P' THEN 'pass'
|
||||
WHEN 'FAIL' THEN 'fail'
|
||||
WHEN 'F' THEN 'fail'
|
||||
WHEN 'REJECT' THEN 'reject'
|
||||
WHEN 'ABORT' THEN 'abort'
|
||||
ELSE NULL
|
||||
END
|
||||
ELSE NULL
|
||||
END
|
||||
END AS outcome,
|
||||
CASE
|
||||
WHEN upper(btrim(s.overall_result)) IN (
|
||||
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
||||
) THEN 'overall_result'
|
||||
WHEN lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD'
|
||||
AND upper(btrim(s.obd_result)) IN (
|
||||
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
||||
) THEN 'utah_obd_proxy'
|
||||
ELSE NULL
|
||||
END AS outcome_label_source
|
||||
-- Only Utah OBD/OBD rows may use utah_obd_proxy, and only for binary
|
||||
-- pass/non-pass analysis. Multiclass analysis must require
|
||||
-- outcome_label_source='overall_result'.
|
||||
FROM data.inspection_search AS s
|
||||
JOIN sampled_vins USING (vin)
|
||||
WHERE s.test_start >= timestamp '2010-01-01'
|
||||
),
|
||||
timestamp_quality AS (
|
||||
SELECT
|
||||
vin,
|
||||
test_start,
|
||||
count(
|
||||
DISTINCT row(
|
||||
source_era,
|
||||
public_county,
|
||||
coalesce(outcome, '<unlabeled>'),
|
||||
coalesce(outcome_label_source, '<unlabeled>')
|
||||
)
|
||||
) AS analytical_variants_at_timestamp
|
||||
FROM base
|
||||
GROUP BY 1, 2
|
||||
),
|
||||
timestamp_groups AS (
|
||||
SELECT
|
||||
base.*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vin, test_start
|
||||
ORDER BY id
|
||||
) AS duplicate_rank
|
||||
FROM base
|
||||
),
|
||||
deduplicated AS (
|
||||
SELECT
|
||||
g.vin,
|
||||
g.id,
|
||||
g.test_start,
|
||||
g.source_era,
|
||||
g.public_county,
|
||||
g.outcome,
|
||||
g.outcome_label_source
|
||||
FROM timestamp_groups AS g
|
||||
JOIN timestamp_quality AS q USING (vin, test_start)
|
||||
WHERE g.duplicate_rank = 1
|
||||
AND q.analytical_variants_at_timestamp = 1
|
||||
),
|
||||
with_gaps AS (
|
||||
SELECT
|
||||
*,
|
||||
lag(test_start) OVER (
|
||||
PARTITION BY vin ORDER BY test_start, id
|
||||
) AS previous_test_start
|
||||
FROM deduplicated
|
||||
),
|
||||
episode_markers AS (
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN previous_test_start IS NULL
|
||||
OR test_start - previous_test_start > interval '30 days'
|
||||
THEN 1 ELSE 0
|
||||
END AS starts_episode
|
||||
FROM with_gaps
|
||||
),
|
||||
numbered AS (
|
||||
SELECT
|
||||
*,
|
||||
sum(starts_episode) OVER (
|
||||
PARTITION BY vin ORDER BY test_start, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS episode_number
|
||||
FROM episode_markers
|
||||
),
|
||||
attempts AS (
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vin, episode_number ORDER BY test_start, id
|
||||
) AS attempt_number
|
||||
FROM numbered
|
||||
),
|
||||
episode_summaries AS (
|
||||
SELECT
|
||||
vin,
|
||||
episode_number,
|
||||
min(test_start) AS episode_start,
|
||||
count(*) AS attempt_count,
|
||||
bool_or(outcome = 'pass') AS eventually_passed
|
||||
FROM attempts
|
||||
GROUP BY 1, 2
|
||||
),
|
||||
episodes AS (
|
||||
SELECT
|
||||
a.vin,
|
||||
a.episode_number,
|
||||
a.test_start AS episode_start,
|
||||
a.source_era,
|
||||
a.public_county,
|
||||
a.outcome AS first_outcome,
|
||||
a.outcome_label_source AS first_outcome_label_source,
|
||||
s.attempt_count,
|
||||
s.eventually_passed
|
||||
FROM attempts AS a
|
||||
JOIN episode_summaries AS s USING (vin, episode_number)
|
||||
WHERE a.attempt_number = 1
|
||||
),
|
||||
sequenced AS (
|
||||
SELECT
|
||||
vin,
|
||||
episode_number,
|
||||
episode_start,
|
||||
source_era,
|
||||
public_county,
|
||||
first_outcome,
|
||||
first_outcome_label_source,
|
||||
lag(episode_start) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_episode_start,
|
||||
lag(first_outcome) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_first_outcome,
|
||||
lag(attempt_count) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_attempt_count,
|
||||
sum(attempt_count) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
) AS prior_event_count
|
||||
FROM episodes
|
||||
),
|
||||
eligible AS (
|
||||
SELECT *
|
||||
FROM sequenced
|
||||
WHERE episode_start >= timestamp '2016-01-01'
|
||||
AND episode_number > 1
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_event_count <= 50
|
||||
)
|
||||
SELECT
|
||||
CASE
|
||||
WHEN grouping(extract(year FROM episode_start)::integer) = 1
|
||||
THEN NULL
|
||||
ELSE extract(year FROM episode_start)::integer
|
||||
END AS target_year,
|
||||
count(*) AS eligible_episodes,
|
||||
count(DISTINCT vin) AS vehicles,
|
||||
count(*) FILTER (WHERE first_outcome = 'pass') AS pass_episodes,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) AS nonpass_episodes,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome_label_source = 'utah_obd_proxy'
|
||||
) AS utah_obd_proxy_episodes,
|
||||
round(
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
)::numeric / nullif(count(*), 0),
|
||||
4
|
||||
) AS nonpass_rate,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY extract(epoch FROM (episode_start-prior_episode_start))/86400.0
|
||||
) AS median_days_since_prior_episode,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY prior_attempt_count
|
||||
) AS median_prior_attempts
|
||||
FROM eligible
|
||||
GROUP BY GROUPING SETS (
|
||||
(),
|
||||
(extract(year FROM episode_start)::integer)
|
||||
)
|
||||
ORDER BY target_year NULLS FIRST;
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,114 @@
|
||||
-- Aggregate-only audit of source-specific inspection result encodings.
|
||||
--
|
||||
-- Values are controlled inspection categories, never identifiers. Counts below
|
||||
-- 100 are suppressed so accidental free-text anomalies cannot be surfaced.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
WITH normalized AS MATERIALIZED (
|
||||
SELECT
|
||||
lower(coalesce(nullif(btrim(county), ''), '<blank/null>')) AS source_era,
|
||||
upper(coalesce(nullif(btrim(overall_result), ''), '<BLANK/NULL>'))
|
||||
AS overall_result,
|
||||
upper(coalesce(nullif(btrim(obd_result), ''), '<BLANK/NULL>'))
|
||||
AS obd_result,
|
||||
lower(coalesce(nullif(btrim(program_type), ''), '<blank/null>'))
|
||||
AS program_type,
|
||||
upper(coalesce(nullif(btrim(test_type), ''), '<BLANK/NULL>'))
|
||||
AS test_type
|
||||
FROM data.inspection_search
|
||||
WHERE test_start >= timestamp '2010-01-01'
|
||||
),
|
||||
overall_counts AS (
|
||||
SELECT
|
||||
'source_overall_result'::text AS audit_section,
|
||||
source_era,
|
||||
overall_result AS value_1,
|
||||
CASE
|
||||
WHEN overall_result IN ('PASS', 'P') THEN 'pass'
|
||||
WHEN overall_result IN ('FAIL', 'F') THEN 'fail'
|
||||
WHEN overall_result = 'REJECT' THEN 'reject'
|
||||
WHEN overall_result = 'ABORT' THEN 'abort'
|
||||
ELSE '<unrecognized>'
|
||||
END::text AS value_2,
|
||||
NULL::text AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
GROUP BY 1, 2, 3, 4
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
utah_unrecognized_context AS (
|
||||
SELECT
|
||||
'utah_unrecognized_context'::text AS audit_section,
|
||||
source_era,
|
||||
overall_result AS value_1,
|
||||
obd_result AS value_2,
|
||||
program_type || ' / ' || test_type AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
WHERE source_era = 'utah'
|
||||
AND overall_result NOT IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
overall_obd_crosscheck AS (
|
||||
SELECT
|
||||
'recognized_overall_vs_obd'::text AS audit_section,
|
||||
source_era,
|
||||
overall_result AS value_1,
|
||||
obd_result AS value_2,
|
||||
program_type || ' / ' || test_type AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
WHERE overall_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
AND obd_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
binary_crosscheck AS (
|
||||
SELECT
|
||||
'binary_overall_vs_obd'::text AS audit_section,
|
||||
source_era,
|
||||
CASE
|
||||
WHEN (overall_result IN ('PASS', 'P'))
|
||||
= (obd_result IN ('PASS', 'P'))
|
||||
THEN 'agree'
|
||||
ELSE 'disagree'
|
||||
END::text AS value_1,
|
||||
NULL::text AS value_2,
|
||||
NULL::text AS value_3,
|
||||
count(*) AS records
|
||||
FROM normalized
|
||||
WHERE overall_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
AND obd_result IN ('PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT')
|
||||
GROUP BY 1, 2, 3
|
||||
HAVING count(*) >= 100
|
||||
),
|
||||
ranked AS (
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY audit_section, source_era
|
||||
ORDER BY records DESC, value_1, value_2, value_3
|
||||
) AS frequency_rank
|
||||
FROM (
|
||||
SELECT * FROM overall_counts
|
||||
UNION ALL
|
||||
SELECT * FROM utah_unrecognized_context
|
||||
UNION ALL
|
||||
SELECT * FROM overall_obd_crosscheck
|
||||
UNION ALL
|
||||
SELECT * FROM binary_crosscheck
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
audit_section,
|
||||
source_era,
|
||||
value_1,
|
||||
value_2,
|
||||
value_3,
|
||||
records
|
||||
FROM ranked
|
||||
WHERE frequency_rank <= 50
|
||||
ORDER BY audit_section, source_era, records DESC;
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,152 @@
|
||||
-- Normalize explicitly all-VARCHAR CSV staging rows, remove exact duplicate
|
||||
-- analytical records, and quarantine timestamps whose event order is ambiguous.
|
||||
|
||||
CREATE OR REPLACE TABLE normalized_events AS
|
||||
SELECT
|
||||
batch_file,
|
||||
batch_kind,
|
||||
batch_start,
|
||||
batch_end,
|
||||
try_cast(internal_event_id AS BIGINT) AS internal_event_id,
|
||||
lower(vehicle_token) AS vehicle_token,
|
||||
try_cast(vehicle_bucket AS INTEGER) AS vehicle_bucket,
|
||||
try_cast(event_ts AS TIMESTAMP) AS event_ts,
|
||||
lower(nullif(trim(source_era), '')) AS source_era,
|
||||
lower(nullif(trim(public_county), '')) AS public_county,
|
||||
lower(nullif(trim(canonical_outcome), '')) AS canonical_outcome,
|
||||
lower(nullif(trim(outcome_label_source), '')) AS outcome_label_source,
|
||||
lower(nullif(trim(program_type), '')) AS program_type,
|
||||
upper(nullif(trim(test_type), '')) AS test_type,
|
||||
upper(nullif(trim(observed_make), '')) AS observed_make,
|
||||
upper(nullif(trim(observed_model), '')) AS observed_model,
|
||||
try_cast(observed_model_year AS INTEGER) AS observed_model_year,
|
||||
CASE
|
||||
WHEN try_cast(internal_event_id AS BIGINT) IS NULL
|
||||
THEN 'invalid_internal_event_id'
|
||||
WHEN vehicle_token IS NULL
|
||||
OR NOT regexp_full_match(lower(vehicle_token), '^[0-9a-f]{64}$')
|
||||
THEN 'invalid_vehicle_token'
|
||||
WHEN try_cast(vehicle_bucket AS INTEGER) IS NULL
|
||||
OR try_cast(vehicle_bucket AS INTEGER) NOT BETWEEN 0 AND 99
|
||||
THEN 'invalid_vehicle_bucket'
|
||||
WHEN try_cast(event_ts AS TIMESTAMP) IS NULL
|
||||
THEN 'invalid_event_timestamp'
|
||||
WHEN try_cast(event_ts AS TIMESTAMP) < TIMESTAMP '2010-01-01'
|
||||
THEN 'pre_2010_timestamp'
|
||||
WHEN try_cast(event_ts AS TIMESTAMP) < batch_start
|
||||
OR try_cast(event_ts AS TIMESTAMP) >= batch_end
|
||||
THEN 'timestamp_outside_manifest_range'
|
||||
WHEN lower(nullif(trim(canonical_outcome), '')) IS NOT NULL
|
||||
AND lower(nullif(trim(canonical_outcome), ''))
|
||||
NOT IN ('pass', 'fail', 'reject', 'abort')
|
||||
THEN 'invalid_canonical_outcome'
|
||||
WHEN lower(nullif(trim(outcome_label_source), '')) IS NOT NULL
|
||||
AND lower(nullif(trim(outcome_label_source), ''))
|
||||
NOT IN ('overall_result', 'utah_obd_proxy')
|
||||
THEN 'invalid_outcome_label_source'
|
||||
WHEN lower(nullif(trim(canonical_outcome), ''))
|
||||
IN ('pass', 'fail', 'reject', 'abort')
|
||||
AND lower(nullif(trim(outcome_label_source), '')) IS NULL
|
||||
THEN 'missing_outcome_label_source'
|
||||
WHEN lower(nullif(trim(outcome_label_source), '')) = 'utah_obd_proxy'
|
||||
AND lower(nullif(trim(source_era), '')) IS DISTINCT FROM 'utah'
|
||||
THEN 'utah_proxy_non_utah_source'
|
||||
WHEN lower(nullif(trim(source_era), '')) IN ('slc', 'slco')
|
||||
AND lower(nullif(trim(public_county), ''))
|
||||
IS DISTINCT FROM 'salt_lake'
|
||||
THEN 'inconsistent_public_county'
|
||||
ELSE NULL
|
||||
END AS invalid_reason
|
||||
FROM stg_events;
|
||||
|
||||
CREATE OR REPLACE TABLE ranked_events AS
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY
|
||||
vehicle_token,
|
||||
vehicle_bucket,
|
||||
event_ts,
|
||||
source_era,
|
||||
public_county,
|
||||
canonical_outcome,
|
||||
outcome_label_source,
|
||||
program_type,
|
||||
test_type,
|
||||
observed_make,
|
||||
observed_model,
|
||||
observed_model_year
|
||||
ORDER BY internal_event_id, batch_file
|
||||
) AS exact_duplicate_rank
|
||||
FROM normalized_events;
|
||||
|
||||
CREATE OR REPLACE TABLE conflicting_timestamps AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
event_ts,
|
||||
count(*) AS distinct_event_count
|
||||
FROM ranked_events
|
||||
WHERE invalid_reason IS NULL
|
||||
AND exact_duplicate_rank = 1
|
||||
GROUP BY vehicle_token, event_ts
|
||||
HAVING count(*) > 1;
|
||||
|
||||
CREATE OR REPLACE TABLE event_exclusions AS
|
||||
SELECT
|
||||
internal_event_id,
|
||||
vehicle_token,
|
||||
event_ts,
|
||||
batch_file,
|
||||
invalid_reason AS exclusion_reason
|
||||
FROM ranked_events
|
||||
WHERE invalid_reason IS NOT NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
internal_event_id,
|
||||
vehicle_token,
|
||||
event_ts,
|
||||
batch_file,
|
||||
'exact_duplicate' AS exclusion_reason
|
||||
FROM ranked_events
|
||||
WHERE invalid_reason IS NULL
|
||||
AND exact_duplicate_rank > 1
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
r.internal_event_id,
|
||||
r.vehicle_token,
|
||||
r.event_ts,
|
||||
r.batch_file,
|
||||
'conflicting_same_timestamp' AS exclusion_reason
|
||||
FROM ranked_events AS r
|
||||
JOIN conflicting_timestamps AS c
|
||||
ON c.vehicle_token = r.vehicle_token
|
||||
AND c.event_ts = r.event_ts
|
||||
WHERE r.invalid_reason IS NULL
|
||||
AND r.exact_duplicate_rank = 1;
|
||||
|
||||
CREATE OR REPLACE TABLE clean_events AS
|
||||
SELECT
|
||||
r.internal_event_id,
|
||||
r.vehicle_token,
|
||||
r.vehicle_bucket,
|
||||
r.event_ts,
|
||||
r.source_era,
|
||||
r.public_county,
|
||||
r.canonical_outcome,
|
||||
r.outcome_label_source,
|
||||
r.program_type,
|
||||
r.test_type,
|
||||
r.observed_make,
|
||||
r.observed_model,
|
||||
r.observed_model_year
|
||||
FROM ranked_events AS r
|
||||
LEFT JOIN conflicting_timestamps AS c
|
||||
ON c.vehicle_token = r.vehicle_token
|
||||
AND c.event_ts = r.event_ts
|
||||
WHERE r.invalid_reason IS NULL
|
||||
AND r.exact_duplicate_rank = 1
|
||||
AND c.vehicle_token IS NULL;
|
||||
@@ -0,0 +1,115 @@
|
||||
-- Construct episodes across the complete, globally ordered input history.
|
||||
-- A gap exactly equal to the configured threshold remains in the same episode.
|
||||
|
||||
CREATE OR REPLACE TABLE sequenced_events AS
|
||||
WITH with_previous AS (
|
||||
SELECT
|
||||
*,
|
||||
lag(event_ts) OVER (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY event_ts, internal_event_id
|
||||
) AS previous_event_ts
|
||||
FROM clean_events
|
||||
),
|
||||
with_markers AS (
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN previous_event_ts IS NULL THEN 1
|
||||
WHEN event_ts - previous_event_ts
|
||||
> (SELECT episode_gap_days FROM build_config)
|
||||
* INTERVAL '1 day'
|
||||
THEN 1
|
||||
ELSE 0
|
||||
END AS starts_episode
|
||||
FROM with_previous
|
||||
),
|
||||
with_episode_number AS (
|
||||
SELECT
|
||||
*,
|
||||
sum(starts_episode) OVER (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY event_ts, internal_event_id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS episode_number
|
||||
FROM with_markers
|
||||
)
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vehicle_token, episode_number
|
||||
ORDER BY event_ts, internal_event_id
|
||||
) AS attempt_number,
|
||||
row_number() OVER (
|
||||
PARTITION BY vehicle_token, episode_number
|
||||
ORDER BY event_ts DESC, internal_event_id DESC
|
||||
) AS reverse_attempt_number
|
||||
FROM with_episode_number;
|
||||
|
||||
CREATE OR REPLACE TABLE episode_daily_activity AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
episode_number,
|
||||
cast(event_ts AS DATE) AS event_date,
|
||||
count(*) AS events_in_day
|
||||
FROM sequenced_events
|
||||
GROUP BY vehicle_token, episode_number, cast(event_ts AS DATE);
|
||||
|
||||
CREATE OR REPLACE TABLE episode_aggregates AS
|
||||
SELECT
|
||||
e.vehicle_token,
|
||||
e.vehicle_bucket,
|
||||
e.episode_number,
|
||||
min(e.event_ts) AS episode_start,
|
||||
max(e.event_ts) AS episode_end,
|
||||
count(*) AS attempt_count,
|
||||
count(e.canonical_outcome) AS labeled_attempt_count,
|
||||
coalesce(bool_or(e.canonical_outcome = 'pass'), false) AS eventually_passed,
|
||||
max(d.events_in_day) AS max_events_in_day,
|
||||
(list(e.observed_make ORDER BY e.event_ts DESC, e.internal_event_id DESC)
|
||||
FILTER (WHERE e.observed_make IS NOT NULL))[1]
|
||||
AS episode_last_observed_make,
|
||||
(list(e.observed_model ORDER BY e.event_ts DESC, e.internal_event_id DESC)
|
||||
FILTER (WHERE e.observed_model IS NOT NULL))[1]
|
||||
AS episode_last_observed_model,
|
||||
(list(e.observed_model_year ORDER BY e.event_ts DESC, e.internal_event_id DESC)
|
||||
FILTER (WHERE e.observed_model_year IS NOT NULL))[1]
|
||||
AS episode_last_observed_model_year
|
||||
FROM sequenced_events AS e
|
||||
JOIN episode_daily_activity AS d
|
||||
ON d.vehicle_token = e.vehicle_token
|
||||
AND d.episode_number = e.episode_number
|
||||
AND d.event_date = cast(e.event_ts AS DATE)
|
||||
GROUP BY e.vehicle_token, e.vehicle_bucket, e.episode_number;
|
||||
|
||||
CREATE OR REPLACE TABLE inspection_episodes AS
|
||||
SELECT
|
||||
a.vehicle_token,
|
||||
a.vehicle_bucket,
|
||||
a.episode_number,
|
||||
a.episode_start,
|
||||
a.episode_end,
|
||||
f.internal_event_id AS first_internal_event_id,
|
||||
f.source_era,
|
||||
f.public_county,
|
||||
f.canonical_outcome AS first_outcome,
|
||||
f.outcome_label_source AS first_outcome_label_source,
|
||||
f.program_type AS first_program_type,
|
||||
f.test_type AS first_test_type,
|
||||
l.canonical_outcome AS final_outcome,
|
||||
a.attempt_count,
|
||||
a.labeled_attempt_count,
|
||||
a.eventually_passed,
|
||||
a.max_events_in_day,
|
||||
a.episode_last_observed_make,
|
||||
a.episode_last_observed_model,
|
||||
a.episode_last_observed_model_year
|
||||
FROM episode_aggregates AS a
|
||||
JOIN sequenced_events AS f
|
||||
ON f.vehicle_token = a.vehicle_token
|
||||
AND f.episode_number = a.episode_number
|
||||
AND f.attempt_number = 1
|
||||
JOIN sequenced_events AS l
|
||||
ON l.vehicle_token = a.vehicle_token
|
||||
AND l.episode_number = a.episode_number
|
||||
AND l.reverse_attempt_number = 1;
|
||||
@@ -0,0 +1,209 @@
|
||||
-- Build one explicitly point-in-time row per episode. All predictors carrying
|
||||
-- history use a window ending at one episode preceding the target.
|
||||
|
||||
CREATE OR REPLACE TABLE episode_history AS
|
||||
WITH history_windows AS (
|
||||
SELECT
|
||||
e.*,
|
||||
count(*) OVER prior_all AS prior_episode_count,
|
||||
coalesce(sum(attempt_count) OVER prior_all, 0) AS prior_total_attempt_count,
|
||||
count(first_outcome) OVER prior_all AS prior_labeled_episode_count,
|
||||
lag(attempt_count) OVER by_vehicle AS prior_attempt_count,
|
||||
lag(first_outcome) OVER by_vehicle AS prior_first_outcome,
|
||||
lag(final_outcome) OVER by_vehicle AS prior_final_outcome,
|
||||
lag(episode_start) OVER by_vehicle AS prior_episode_start,
|
||||
max(CASE
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort')
|
||||
THEN episode_start
|
||||
END) OVER prior_all AS prior_adverse_episode_start,
|
||||
count(*) FILTER (WHERE first_outcome = 'pass') OVER prior_all
|
||||
AS prior_pass_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'fail') OVER prior_all
|
||||
AS prior_fail_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'reject') OVER prior_all
|
||||
AS prior_reject_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'abort') OVER prior_all
|
||||
AS prior_abort_count,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) OVER prior_all AS prior_nonpass_count,
|
||||
count(first_outcome) OVER prior_three AS last3_labeled_episode_count,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) OVER prior_three AS last3_nonpass_count,
|
||||
max(max_events_in_day) OVER prior_all AS prior_max_events_in_day,
|
||||
arg_max(episode_last_observed_make, episode_number)
|
||||
FILTER (WHERE episode_last_observed_make IS NOT NULL)
|
||||
OVER prior_all AS last_observed_make,
|
||||
arg_max(episode_last_observed_model, episode_number)
|
||||
FILTER (WHERE episode_last_observed_model IS NOT NULL)
|
||||
OVER prior_all AS last_observed_model,
|
||||
arg_max(episode_last_observed_model_year, episode_number)
|
||||
FILTER (WHERE episode_last_observed_model_year IS NOT NULL)
|
||||
OVER prior_all AS last_observed_model_year
|
||||
FROM inspection_episodes AS e
|
||||
WINDOW
|
||||
by_vehicle AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
),
|
||||
prior_all AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
),
|
||||
prior_three AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
ROWS BETWEEN 3 PRECEDING AND 1 PRECEDING
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN prior_episode_start IS NULL THEN NULL
|
||||
ELSE extract(epoch FROM (episode_start - prior_episode_start)) / 86400.0
|
||||
END AS days_since_prior_episode,
|
||||
CASE
|
||||
WHEN prior_adverse_episode_start IS NULL THEN NULL
|
||||
ELSE extract(epoch FROM (episode_start - prior_adverse_episode_start))
|
||||
/ 86400.0
|
||||
END AS days_since_prior_adverse,
|
||||
prior_nonpass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
||||
AS prior_nonpass_rate,
|
||||
prior_pass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
||||
AS prior_pass_rate,
|
||||
last3_nonpass_count::DOUBLE / nullif(last3_labeled_episode_count, 0)
|
||||
AS last3_nonpass_rate
|
||||
FROM history_windows;
|
||||
|
||||
CREATE OR REPLACE TABLE feature_mart AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
vehicle_bucket,
|
||||
vehicle_bucket < 10 AS is_vin_audit,
|
||||
episode_number::BIGINT AS episode_number,
|
||||
episode_start,
|
||||
first_outcome,
|
||||
first_outcome_label_source AS target_outcome_label_source,
|
||||
CASE
|
||||
WHEN first_outcome = 'pass' THEN 0
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
|
||||
ELSE NULL
|
||||
END::INTEGER AS target_nonpass,
|
||||
episode_start >= TIMESTAMP '2016-01-01'
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_episode_count >= 1
|
||||
AND prior_total_attempt_count <= 50
|
||||
AND coalesce(prior_max_events_in_day, 0) <= 4
|
||||
AS eligible_returning_target,
|
||||
CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
|
||||
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
|
||||
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
|
||||
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
|
||||
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
|
||||
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
|
||||
ELSE 'out_of_scope'
|
||||
END AS temporal_partition,
|
||||
CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'before_target_period'
|
||||
WHEN first_outcome IS NULL THEN 'unlabeled_first_outcome'
|
||||
WHEN prior_episode_count < 1 THEN 'cold_start'
|
||||
WHEN prior_total_attempt_count > 50 THEN 'prior_event_count_over_50'
|
||||
WHEN coalesce(prior_max_events_in_day, 0) > 4
|
||||
THEN 'prior_daily_activity_over_4'
|
||||
ELSE NULL
|
||||
END AS eligibility_exclusion_reason,
|
||||
public_county,
|
||||
source_era,
|
||||
CASE
|
||||
WHEN month(episode_start) IN (12, 1, 2) THEN 'winter'
|
||||
WHEN month(episode_start) IN (3, 4, 5) THEN 'spring'
|
||||
WHEN month(episode_start) IN (6, 7, 8) THEN 'summer'
|
||||
ELSE 'fall'
|
||||
END AS target_season,
|
||||
CASE
|
||||
WHEN last_observed_model_year BETWEEN 1886 AND year(episode_start) + 1
|
||||
THEN greatest(year(episode_start) - last_observed_model_year, 0)
|
||||
ELSE NULL
|
||||
END::INTEGER AS vehicle_age,
|
||||
prior_episode_count,
|
||||
prior_total_attempt_count::BIGINT AS prior_total_attempt_count,
|
||||
prior_attempt_count,
|
||||
days_since_prior_episode,
|
||||
days_since_prior_adverse,
|
||||
prior_nonpass_rate,
|
||||
prior_first_outcome,
|
||||
prior_final_outcome,
|
||||
last_observed_make,
|
||||
last_observed_model,
|
||||
last_observed_model_year,
|
||||
prior_labeled_episode_count,
|
||||
prior_pass_count,
|
||||
prior_fail_count,
|
||||
prior_reject_count,
|
||||
prior_abort_count,
|
||||
prior_nonpass_count,
|
||||
prior_pass_rate,
|
||||
last3_labeled_episode_count,
|
||||
last3_nonpass_count,
|
||||
last3_nonpass_rate,
|
||||
prior_max_events_in_day,
|
||||
prior_first_outcome IS NULL AS prior_first_outcome_missing,
|
||||
last_observed_model_year IS NULL AS prior_model_year_missing,
|
||||
(SELECT source_data_kind FROM build_config) AS source_data_kind,
|
||||
(SELECT population_estimate_allowed FROM build_config)
|
||||
AS population_estimate_allowed
|
||||
FROM episode_history;
|
||||
|
||||
CREATE OR REPLACE TABLE cohort_flow AS
|
||||
SELECT
|
||||
'event' AS grain,
|
||||
'input' AS stage,
|
||||
'all_rows' AS reason,
|
||||
count(*) AS observation_count,
|
||||
count(DISTINCT try_cast(vehicle_token AS VARCHAR)) AS vehicle_count
|
||||
FROM stg_events
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'event',
|
||||
'excluded',
|
||||
exclusion_reason,
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM event_exclusions
|
||||
GROUP BY exclusion_reason
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'event',
|
||||
'accepted',
|
||||
'clean_events',
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM clean_events
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode',
|
||||
'constructed',
|
||||
'all_episodes',
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM inspection_episodes
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode',
|
||||
'eligible_returning_target',
|
||||
coalesce(eligibility_exclusion_reason, 'eligible'),
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM feature_mart
|
||||
GROUP BY coalesce(eligibility_exclusion_reason, 'eligible');
|
||||
@@ -0,0 +1,155 @@
|
||||
-- Every row reports a count of invariant violations. The Python driver refuses
|
||||
-- to publish the mart unless every count is zero.
|
||||
|
||||
CREATE OR REPLACE TABLE mart_validation AS
|
||||
SELECT
|
||||
'clean_internal_event_id_unique' AS check_name,
|
||||
count(*) AS violation_count
|
||||
FROM (
|
||||
SELECT internal_event_id
|
||||
FROM clean_events
|
||||
GROUP BY internal_event_id
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode_key_unique',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT vehicle_token, episode_number
|
||||
FROM inspection_episodes
|
||||
GROUP BY vehicle_token, episode_number
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'mart_key_unique',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT vehicle_token, episode_number
|
||||
FROM feature_mart
|
||||
GROUP BY vehicle_token, episode_number
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode_attempts_reconcile',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT
|
||||
e.vehicle_token,
|
||||
e.episode_number
|
||||
FROM inspection_episodes AS e
|
||||
JOIN (
|
||||
SELECT vehicle_token, episode_number, count(*) AS actual_attempts
|
||||
FROM sequenced_events
|
||||
GROUP BY vehicle_token, episode_number
|
||||
) AS a USING (vehicle_token, episode_number)
|
||||
WHERE e.attempt_count <> a.actual_attempts
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode_gap_strictly_greater_than_threshold',
|
||||
count(*)
|
||||
FROM (
|
||||
SELECT
|
||||
episode_start,
|
||||
lag(episode_end) OVER (
|
||||
PARTITION BY vehicle_token ORDER BY episode_number
|
||||
) AS prior_episode_end
|
||||
FROM inspection_episodes
|
||||
) AS gaps
|
||||
WHERE prior_episode_end IS NOT NULL
|
||||
AND episode_start - prior_episode_end
|
||||
<= (SELECT episode_gap_days FROM build_config) * INTERVAL '1 day'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'prior_episode_count_point_in_time',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE prior_episode_count <> episode_number - 1
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'target_mapping_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE target_nonpass IS DISTINCT FROM CASE
|
||||
WHEN first_outcome = 'pass' THEN 0
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
|
||||
ELSE NULL
|
||||
END
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'target_outcome_label_source_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE (first_outcome IS NOT NULL AND (
|
||||
target_outcome_label_source IS NULL
|
||||
OR target_outcome_label_source
|
||||
NOT IN ('overall_result', 'utah_obd_proxy')
|
||||
))
|
||||
OR (target_outcome_label_source = 'utah_obd_proxy'
|
||||
AND source_era IS DISTINCT FROM 'utah')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'raw_obd_result_absent_from_mart',
|
||||
count(*)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'main'
|
||||
AND table_name = 'feature_mart'
|
||||
AND lower(column_name) = 'obd_result'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'audit_bucket_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE vehicle_bucket NOT BETWEEN 0 AND 99
|
||||
OR is_vin_audit IS DISTINCT FROM (vehicle_bucket < 10)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'eligibility_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE eligible_returning_target IS DISTINCT FROM (
|
||||
episode_start >= TIMESTAMP '2016-01-01'
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_episode_count >= 1
|
||||
AND prior_total_attempt_count <= 50
|
||||
AND coalesce(prior_max_events_in_day, 0) <= 4
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'temporal_partition_consistent',
|
||||
count(*)
|
||||
FROM feature_mart
|
||||
WHERE temporal_partition IS DISTINCT FROM CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
|
||||
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
|
||||
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
|
||||
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
|
||||
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
|
||||
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
|
||||
ELSE 'out_of_scope'
|
||||
END;
|
||||
Reference in New Issue
Block a user