Centralized_I-M_Database/SummerProject2026
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial code

Browse Source
This commit is contained in:
kevinbell
2026-07-15 17:55:53 -06:00
parent 0952a7ffce
commit 05729fc6de
53 changed files with 12965 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sql/local/20_stage_events.sql
+152
View File
@@ -0,0 +1,152 @@
-- Normalize explicitly all-VARCHAR CSV staging rows, remove exact duplicate
-- analytical records, and quarantine timestamps whose event order is ambiguous.
CREATE OR REPLACE TABLE normalized_events AS
SELECT
batch_file,
batch_kind,
batch_start,
batch_end,
try_cast(internal_event_id AS BIGINT) AS internal_event_id,
lower(vehicle_token) AS vehicle_token,
try_cast(vehicle_bucket AS INTEGER) AS vehicle_bucket,
try_cast(event_ts AS TIMESTAMP) AS event_ts,
lower(nullif(trim(source_era), '')) AS source_era,
lower(nullif(trim(public_county), '')) AS public_county,
lower(nullif(trim(canonical_outcome), '')) AS canonical_outcome,
lower(nullif(trim(outcome_label_source), '')) AS outcome_label_source,
lower(nullif(trim(program_type), '')) AS program_type,
upper(nullif(trim(test_type), '')) AS test_type,
upper(nullif(trim(observed_make), '')) AS observed_make,
upper(nullif(trim(observed_model), '')) AS observed_model,
try_cast(observed_model_year AS INTEGER) AS observed_model_year,
CASE
WHEN try_cast(internal_event_id AS BIGINT) IS NULL
THEN 'invalid_internal_event_id'
WHEN vehicle_token IS NULL
OR NOT regexp_full_match(lower(vehicle_token), '^[0-9a-f]{64}$')
THEN 'invalid_vehicle_token'
WHEN try_cast(vehicle_bucket AS INTEGER) IS NULL
OR try_cast(vehicle_bucket AS INTEGER) NOT BETWEEN 0 AND 99
THEN 'invalid_vehicle_bucket'
WHEN try_cast(event_ts AS TIMESTAMP) IS NULL
THEN 'invalid_event_timestamp'
WHEN try_cast(event_ts AS TIMESTAMP) < TIMESTAMP '2010-01-01'
THEN 'pre_2010_timestamp'
WHEN try_cast(event_ts AS TIMESTAMP) < batch_start
OR try_cast(event_ts AS TIMESTAMP) >= batch_end
THEN 'timestamp_outside_manifest_range'
WHEN lower(nullif(trim(canonical_outcome), '')) IS NOT NULL
AND lower(nullif(trim(canonical_outcome), ''))
NOT IN ('pass', 'fail', 'reject', 'abort')
THEN 'invalid_canonical_outcome'
WHEN lower(nullif(trim(outcome_label_source), '')) IS NOT NULL
AND lower(nullif(trim(outcome_label_source), ''))
NOT IN ('overall_result', 'utah_obd_proxy')
THEN 'invalid_outcome_label_source'
WHEN lower(nullif(trim(canonical_outcome), ''))
IN ('pass', 'fail', 'reject', 'abort')
AND lower(nullif(trim(outcome_label_source), '')) IS NULL
THEN 'missing_outcome_label_source'
WHEN lower(nullif(trim(outcome_label_source), '')) = 'utah_obd_proxy'
AND lower(nullif(trim(source_era), '')) IS DISTINCT FROM 'utah'
THEN 'utah_proxy_non_utah_source'
WHEN lower(nullif(trim(source_era), '')) IN ('slc', 'slco')
AND lower(nullif(trim(public_county), ''))
IS DISTINCT FROM 'salt_lake'
THEN 'inconsistent_public_county'
ELSE NULL
END AS invalid_reason
FROM stg_events;
CREATE OR REPLACE TABLE ranked_events AS
SELECT
*,
row_number() OVER (
PARTITION BY
vehicle_token,
vehicle_bucket,
event_ts,
source_era,
public_county,
canonical_outcome,
outcome_label_source,
program_type,
test_type,
observed_make,
observed_model,
observed_model_year
ORDER BY internal_event_id, batch_file
) AS exact_duplicate_rank
FROM normalized_events;
CREATE OR REPLACE TABLE conflicting_timestamps AS
SELECT
vehicle_token,
event_ts,
count(*) AS distinct_event_count
FROM ranked_events
WHERE invalid_reason IS NULL
AND exact_duplicate_rank = 1
GROUP BY vehicle_token, event_ts
HAVING count(*) > 1;
CREATE OR REPLACE TABLE event_exclusions AS
SELECT
internal_event_id,
vehicle_token,
event_ts,
batch_file,
invalid_reason AS exclusion_reason
FROM ranked_events
WHERE invalid_reason IS NOT NULL
UNION ALL
SELECT
internal_event_id,
vehicle_token,
event_ts,
batch_file,
'exact_duplicate' AS exclusion_reason
FROM ranked_events
WHERE invalid_reason IS NULL
AND exact_duplicate_rank > 1
UNION ALL
SELECT
r.internal_event_id,
r.vehicle_token,
r.event_ts,
r.batch_file,
'conflicting_same_timestamp' AS exclusion_reason
FROM ranked_events AS r
JOIN conflicting_timestamps AS c
ON c.vehicle_token = r.vehicle_token
AND c.event_ts = r.event_ts
WHERE r.invalid_reason IS NULL
AND r.exact_duplicate_rank = 1;
CREATE OR REPLACE TABLE clean_events AS
SELECT
r.internal_event_id,
r.vehicle_token,
r.vehicle_bucket,
r.event_ts,
r.source_era,
r.public_county,
r.canonical_outcome,
r.outcome_label_source,
r.program_type,
r.test_type,
r.observed_make,
r.observed_model,
r.observed_model_year
FROM ranked_events AS r
LEFT JOIN conflicting_timestamps AS c
ON c.vehicle_token = r.vehicle_token
AND c.event_ts = r.event_ts
WHERE r.invalid_reason IS NULL
AND r.exact_duplicate_rank = 1
AND c.vehicle_token IS NULL;
sql/local/21_build_episodes.sql
+115
View File
@@ -0,0 +1,115 @@
-- Construct episodes across the complete, globally ordered input history.
-- A gap exactly equal to the configured threshold remains in the same episode.
CREATE OR REPLACE TABLE sequenced_events AS
WITH with_previous AS (
SELECT
*,
lag(event_ts) OVER (
PARTITION BY vehicle_token
ORDER BY event_ts, internal_event_id
) AS previous_event_ts
FROM clean_events
),
with_markers AS (
SELECT
*,
CASE
WHEN previous_event_ts IS NULL THEN 1
WHEN event_ts - previous_event_ts
> (SELECT episode_gap_days FROM build_config)
* INTERVAL '1 day'
THEN 1
ELSE 0
END AS starts_episode
FROM with_previous
),
with_episode_number AS (
SELECT
*,
sum(starts_episode) OVER (
PARTITION BY vehicle_token
ORDER BY event_ts, internal_event_id
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
) AS episode_number
FROM with_markers
)
SELECT
*,
row_number() OVER (
PARTITION BY vehicle_token, episode_number
ORDER BY event_ts, internal_event_id
) AS attempt_number,
row_number() OVER (
PARTITION BY vehicle_token, episode_number
ORDER BY event_ts DESC, internal_event_id DESC
) AS reverse_attempt_number
FROM with_episode_number;
CREATE OR REPLACE TABLE episode_daily_activity AS
SELECT
vehicle_token,
episode_number,
cast(event_ts AS DATE) AS event_date,
count(*) AS events_in_day
FROM sequenced_events
GROUP BY vehicle_token, episode_number, cast(event_ts AS DATE);
CREATE OR REPLACE TABLE episode_aggregates AS
SELECT
e.vehicle_token,
e.vehicle_bucket,
e.episode_number,
min(e.event_ts) AS episode_start,
max(e.event_ts) AS episode_end,
count(*) AS attempt_count,
count(e.canonical_outcome) AS labeled_attempt_count,
coalesce(bool_or(e.canonical_outcome = 'pass'), false) AS eventually_passed,
max(d.events_in_day) AS max_events_in_day,
(list(e.observed_make ORDER BY e.event_ts DESC, e.internal_event_id DESC)
FILTER (WHERE e.observed_make IS NOT NULL))[1]
AS episode_last_observed_make,
(list(e.observed_model ORDER BY e.event_ts DESC, e.internal_event_id DESC)
FILTER (WHERE e.observed_model IS NOT NULL))[1]
AS episode_last_observed_model,
(list(e.observed_model_year ORDER BY e.event_ts DESC, e.internal_event_id DESC)
FILTER (WHERE e.observed_model_year IS NOT NULL))[1]
AS episode_last_observed_model_year
FROM sequenced_events AS e
JOIN episode_daily_activity AS d
ON d.vehicle_token = e.vehicle_token
AND d.episode_number = e.episode_number
AND d.event_date = cast(e.event_ts AS DATE)
GROUP BY e.vehicle_token, e.vehicle_bucket, e.episode_number;
CREATE OR REPLACE TABLE inspection_episodes AS
SELECT
a.vehicle_token,
a.vehicle_bucket,
a.episode_number,
a.episode_start,
a.episode_end,
f.internal_event_id AS first_internal_event_id,
f.source_era,
f.public_county,
f.canonical_outcome AS first_outcome,
f.outcome_label_source AS first_outcome_label_source,
f.program_type AS first_program_type,
f.test_type AS first_test_type,
l.canonical_outcome AS final_outcome,
a.attempt_count,
a.labeled_attempt_count,
a.eventually_passed,
a.max_events_in_day,
a.episode_last_observed_make,
a.episode_last_observed_model,
a.episode_last_observed_model_year
FROM episode_aggregates AS a
JOIN sequenced_events AS f
ON f.vehicle_token = a.vehicle_token
AND f.episode_number = a.episode_number
AND f.attempt_number = 1
JOIN sequenced_events AS l
ON l.vehicle_token = a.vehicle_token
AND l.episode_number = a.episode_number
AND l.reverse_attempt_number = 1;
sql/local/22_build_features.sql
+209
View File
@@ -0,0 +1,209 @@
-- Build one explicitly point-in-time row per episode. All predictors carrying
-- history use a window ending at one episode preceding the target.
CREATE OR REPLACE TABLE episode_history AS
WITH history_windows AS (
SELECT
e.*,
count(*) OVER prior_all AS prior_episode_count,
coalesce(sum(attempt_count) OVER prior_all, 0) AS prior_total_attempt_count,
count(first_outcome) OVER prior_all AS prior_labeled_episode_count,
lag(attempt_count) OVER by_vehicle AS prior_attempt_count,
lag(first_outcome) OVER by_vehicle AS prior_first_outcome,
lag(final_outcome) OVER by_vehicle AS prior_final_outcome,
lag(episode_start) OVER by_vehicle AS prior_episode_start,
max(CASE
WHEN first_outcome IN ('fail', 'reject', 'abort')
THEN episode_start
END) OVER prior_all AS prior_adverse_episode_start,
count(*) FILTER (WHERE first_outcome = 'pass') OVER prior_all
AS prior_pass_count,
count(*) FILTER (WHERE first_outcome = 'fail') OVER prior_all
AS prior_fail_count,
count(*) FILTER (WHERE first_outcome = 'reject') OVER prior_all
AS prior_reject_count,
count(*) FILTER (WHERE first_outcome = 'abort') OVER prior_all
AS prior_abort_count,
count(*) FILTER (
WHERE first_outcome IN ('fail', 'reject', 'abort')
) OVER prior_all AS prior_nonpass_count,
count(first_outcome) OVER prior_three AS last3_labeled_episode_count,
count(*) FILTER (
WHERE first_outcome IN ('fail', 'reject', 'abort')
) OVER prior_three AS last3_nonpass_count,
max(max_events_in_day) OVER prior_all AS prior_max_events_in_day,
arg_max(episode_last_observed_make, episode_number)
FILTER (WHERE episode_last_observed_make IS NOT NULL)
OVER prior_all AS last_observed_make,
arg_max(episode_last_observed_model, episode_number)
FILTER (WHERE episode_last_observed_model IS NOT NULL)
OVER prior_all AS last_observed_model,
arg_max(episode_last_observed_model_year, episode_number)
FILTER (WHERE episode_last_observed_model_year IS NOT NULL)
OVER prior_all AS last_observed_model_year
FROM inspection_episodes AS e
WINDOW
by_vehicle AS (
PARTITION BY vehicle_token
ORDER BY episode_number
),
prior_all AS (
PARTITION BY vehicle_token
ORDER BY episode_number
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
),
prior_three AS (
PARTITION BY vehicle_token
ORDER BY episode_number
ROWS BETWEEN 3 PRECEDING AND 1 PRECEDING
)
)
SELECT
*,
CASE
WHEN prior_episode_start IS NULL THEN NULL
ELSE extract(epoch FROM (episode_start - prior_episode_start)) / 86400.0
END AS days_since_prior_episode,
CASE
WHEN prior_adverse_episode_start IS NULL THEN NULL
ELSE extract(epoch FROM (episode_start - prior_adverse_episode_start))
/ 86400.0
END AS days_since_prior_adverse,
prior_nonpass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
AS prior_nonpass_rate,
prior_pass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
AS prior_pass_rate,
last3_nonpass_count::DOUBLE / nullif(last3_labeled_episode_count, 0)
AS last3_nonpass_rate
FROM history_windows;
CREATE OR REPLACE TABLE feature_mart AS
SELECT
vehicle_token,
vehicle_bucket,
vehicle_bucket < 10 AS is_vin_audit,
episode_number::BIGINT AS episode_number,
episode_start,
first_outcome,
first_outcome_label_source AS target_outcome_label_source,
CASE
WHEN first_outcome = 'pass' THEN 0
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
ELSE NULL
END::INTEGER AS target_nonpass,
episode_start >= TIMESTAMP '2016-01-01'
AND first_outcome IS NOT NULL
AND prior_episode_count >= 1
AND prior_total_attempt_count <= 50
AND coalesce(prior_max_events_in_day, 0) <= 4
AS eligible_returning_target,
CASE
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
ELSE 'out_of_scope'
END AS temporal_partition,
CASE
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'before_target_period'
WHEN first_outcome IS NULL THEN 'unlabeled_first_outcome'
WHEN prior_episode_count < 1 THEN 'cold_start'
WHEN prior_total_attempt_count > 50 THEN 'prior_event_count_over_50'
WHEN coalesce(prior_max_events_in_day, 0) > 4
THEN 'prior_daily_activity_over_4'
ELSE NULL
END AS eligibility_exclusion_reason,
public_county,
source_era,
CASE
WHEN month(episode_start) IN (12, 1, 2) THEN 'winter'
WHEN month(episode_start) IN (3, 4, 5) THEN 'spring'
WHEN month(episode_start) IN (6, 7, 8) THEN 'summer'
ELSE 'fall'
END AS target_season,
CASE
WHEN last_observed_model_year BETWEEN 1886 AND year(episode_start) + 1
THEN greatest(year(episode_start) - last_observed_model_year, 0)
ELSE NULL
END::INTEGER AS vehicle_age,
prior_episode_count,
prior_total_attempt_count::BIGINT AS prior_total_attempt_count,
prior_attempt_count,
days_since_prior_episode,
days_since_prior_adverse,
prior_nonpass_rate,
prior_first_outcome,
prior_final_outcome,
last_observed_make,
last_observed_model,
last_observed_model_year,
prior_labeled_episode_count,
prior_pass_count,
prior_fail_count,
prior_reject_count,
prior_abort_count,
prior_nonpass_count,
prior_pass_rate,
last3_labeled_episode_count,
last3_nonpass_count,
last3_nonpass_rate,
prior_max_events_in_day,
prior_first_outcome IS NULL AS prior_first_outcome_missing,
last_observed_model_year IS NULL AS prior_model_year_missing,
(SELECT source_data_kind FROM build_config) AS source_data_kind,
(SELECT population_estimate_allowed FROM build_config)
AS population_estimate_allowed
FROM episode_history;
CREATE OR REPLACE TABLE cohort_flow AS
SELECT
'event' AS grain,
'input' AS stage,
'all_rows' AS reason,
count(*) AS observation_count,
count(DISTINCT try_cast(vehicle_token AS VARCHAR)) AS vehicle_count
FROM stg_events
UNION ALL
SELECT
'event',
'excluded',
exclusion_reason,
count(*),
count(DISTINCT vehicle_token)
FROM event_exclusions
GROUP BY exclusion_reason
UNION ALL
SELECT
'event',
'accepted',
'clean_events',
count(*),
count(DISTINCT vehicle_token)
FROM clean_events
UNION ALL
SELECT
'episode',
'constructed',
'all_episodes',
count(*),
count(DISTINCT vehicle_token)
FROM inspection_episodes
UNION ALL
SELECT
'episode',
'eligible_returning_target',
coalesce(eligibility_exclusion_reason, 'eligible'),
count(*),
count(DISTINCT vehicle_token)
FROM feature_mart
GROUP BY coalesce(eligibility_exclusion_reason, 'eligible');
sql/local/23_validate_mart.sql
+155
View File
@@ -0,0 +1,155 @@
-- Every row reports a count of invariant violations. The Python driver refuses
-- to publish the mart unless every count is zero.
CREATE OR REPLACE TABLE mart_validation AS
SELECT
'clean_internal_event_id_unique' AS check_name,
count(*) AS violation_count
FROM (
SELECT internal_event_id
FROM clean_events
GROUP BY internal_event_id
HAVING count(*) > 1
)
UNION ALL
SELECT
'episode_key_unique',
count(*)
FROM (
SELECT vehicle_token, episode_number
FROM inspection_episodes
GROUP BY vehicle_token, episode_number
HAVING count(*) > 1
)
UNION ALL
SELECT
'mart_key_unique',
count(*)
FROM (
SELECT vehicle_token, episode_number
FROM feature_mart
GROUP BY vehicle_token, episode_number
HAVING count(*) > 1
)
UNION ALL
SELECT
'episode_attempts_reconcile',
count(*)
FROM (
SELECT
e.vehicle_token,
e.episode_number
FROM inspection_episodes AS e
JOIN (
SELECT vehicle_token, episode_number, count(*) AS actual_attempts
FROM sequenced_events
GROUP BY vehicle_token, episode_number
) AS a USING (vehicle_token, episode_number)
WHERE e.attempt_count <> a.actual_attempts
)
UNION ALL
SELECT
'episode_gap_strictly_greater_than_threshold',
count(*)
FROM (
SELECT
episode_start,
lag(episode_end) OVER (
PARTITION BY vehicle_token ORDER BY episode_number
) AS prior_episode_end
FROM inspection_episodes
) AS gaps
WHERE prior_episode_end IS NOT NULL
AND episode_start - prior_episode_end
<= (SELECT episode_gap_days FROM build_config) * INTERVAL '1 day'
UNION ALL
SELECT
'prior_episode_count_point_in_time',
count(*)
FROM feature_mart
WHERE prior_episode_count <> episode_number - 1
UNION ALL
SELECT
'target_mapping_consistent',
count(*)
FROM feature_mart
WHERE target_nonpass IS DISTINCT FROM CASE
WHEN first_outcome = 'pass' THEN 0
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
ELSE NULL
END
UNION ALL
SELECT
'target_outcome_label_source_consistent',
count(*)
FROM feature_mart
WHERE (first_outcome IS NOT NULL AND (
target_outcome_label_source IS NULL
OR target_outcome_label_source
NOT IN ('overall_result', 'utah_obd_proxy')
))
OR (target_outcome_label_source = 'utah_obd_proxy'
AND source_era IS DISTINCT FROM 'utah')
UNION ALL
SELECT
'raw_obd_result_absent_from_mart',
count(*)
FROM information_schema.columns
WHERE table_schema = 'main'
AND table_name = 'feature_mart'
AND lower(column_name) = 'obd_result'
UNION ALL
SELECT
'audit_bucket_consistent',
count(*)
FROM feature_mart
WHERE vehicle_bucket NOT BETWEEN 0 AND 99
OR is_vin_audit IS DISTINCT FROM (vehicle_bucket < 10)
UNION ALL
SELECT
'eligibility_consistent',
count(*)
FROM feature_mart
WHERE eligible_returning_target IS DISTINCT FROM (
episode_start >= TIMESTAMP '2016-01-01'
AND first_outcome IS NOT NULL
AND prior_episode_count >= 1
AND prior_total_attempt_count <= 50
AND coalesce(prior_max_events_in_day, 0) <= 4
)
UNION ALL
SELECT
'temporal_partition_consistent',
count(*)
FROM feature_mart
WHERE temporal_partition IS DISTINCT FROM CASE
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
ELSE 'out_of_scope'
END;