210 lines
7.3 KiB
SQL
210 lines
7.3 KiB
SQL
-- Build one explicitly point-in-time row per episode. All predictors carrying
|
|
-- history use a window ending at one episode preceding the target.
|
|
|
|
CREATE OR REPLACE TABLE episode_history AS
|
|
WITH history_windows AS (
|
|
SELECT
|
|
e.*,
|
|
count(*) OVER prior_all AS prior_episode_count,
|
|
coalesce(sum(attempt_count) OVER prior_all, 0) AS prior_total_attempt_count,
|
|
count(first_outcome) OVER prior_all AS prior_labeled_episode_count,
|
|
lag(attempt_count) OVER by_vehicle AS prior_attempt_count,
|
|
lag(first_outcome) OVER by_vehicle AS prior_first_outcome,
|
|
lag(final_outcome) OVER by_vehicle AS prior_final_outcome,
|
|
lag(episode_start) OVER by_vehicle AS prior_episode_start,
|
|
max(CASE
|
|
WHEN first_outcome IN ('fail', 'reject', 'abort')
|
|
THEN episode_start
|
|
END) OVER prior_all AS prior_adverse_episode_start,
|
|
count(*) FILTER (WHERE first_outcome = 'pass') OVER prior_all
|
|
AS prior_pass_count,
|
|
count(*) FILTER (WHERE first_outcome = 'fail') OVER prior_all
|
|
AS prior_fail_count,
|
|
count(*) FILTER (WHERE first_outcome = 'reject') OVER prior_all
|
|
AS prior_reject_count,
|
|
count(*) FILTER (WHERE first_outcome = 'abort') OVER prior_all
|
|
AS prior_abort_count,
|
|
count(*) FILTER (
|
|
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
|
) OVER prior_all AS prior_nonpass_count,
|
|
count(first_outcome) OVER prior_three AS last3_labeled_episode_count,
|
|
count(*) FILTER (
|
|
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
|
) OVER prior_three AS last3_nonpass_count,
|
|
max(max_events_in_day) OVER prior_all AS prior_max_events_in_day,
|
|
arg_max(episode_last_observed_make, episode_number)
|
|
FILTER (WHERE episode_last_observed_make IS NOT NULL)
|
|
OVER prior_all AS last_observed_make,
|
|
arg_max(episode_last_observed_model, episode_number)
|
|
FILTER (WHERE episode_last_observed_model IS NOT NULL)
|
|
OVER prior_all AS last_observed_model,
|
|
arg_max(episode_last_observed_model_year, episode_number)
|
|
FILTER (WHERE episode_last_observed_model_year IS NOT NULL)
|
|
OVER prior_all AS last_observed_model_year
|
|
FROM inspection_episodes AS e
|
|
WINDOW
|
|
by_vehicle AS (
|
|
PARTITION BY vehicle_token
|
|
ORDER BY episode_number
|
|
),
|
|
prior_all AS (
|
|
PARTITION BY vehicle_token
|
|
ORDER BY episode_number
|
|
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
|
),
|
|
prior_three AS (
|
|
PARTITION BY vehicle_token
|
|
ORDER BY episode_number
|
|
ROWS BETWEEN 3 PRECEDING AND 1 PRECEDING
|
|
)
|
|
)
|
|
SELECT
|
|
*,
|
|
CASE
|
|
WHEN prior_episode_start IS NULL THEN NULL
|
|
ELSE extract(epoch FROM (episode_start - prior_episode_start)) / 86400.0
|
|
END AS days_since_prior_episode,
|
|
CASE
|
|
WHEN prior_adverse_episode_start IS NULL THEN NULL
|
|
ELSE extract(epoch FROM (episode_start - prior_adverse_episode_start))
|
|
/ 86400.0
|
|
END AS days_since_prior_adverse,
|
|
prior_nonpass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
|
AS prior_nonpass_rate,
|
|
prior_pass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
|
AS prior_pass_rate,
|
|
last3_nonpass_count::DOUBLE / nullif(last3_labeled_episode_count, 0)
|
|
AS last3_nonpass_rate
|
|
FROM history_windows;
|
|
|
|
CREATE OR REPLACE TABLE feature_mart AS
|
|
SELECT
|
|
vehicle_token,
|
|
vehicle_bucket,
|
|
vehicle_bucket < 10 AS is_vin_audit,
|
|
episode_number::BIGINT AS episode_number,
|
|
episode_start,
|
|
first_outcome,
|
|
first_outcome_label_source AS target_outcome_label_source,
|
|
CASE
|
|
WHEN first_outcome = 'pass' THEN 0
|
|
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
|
|
ELSE NULL
|
|
END::INTEGER AS target_nonpass,
|
|
episode_start >= TIMESTAMP '2016-01-01'
|
|
AND first_outcome IS NOT NULL
|
|
AND prior_episode_count >= 1
|
|
AND prior_total_attempt_count <= 50
|
|
AND coalesce(prior_max_events_in_day, 0) <= 4
|
|
AS eligible_returning_target,
|
|
CASE
|
|
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
|
|
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
|
|
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
|
|
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
|
|
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
|
|
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
|
|
ELSE 'out_of_scope'
|
|
END AS temporal_partition,
|
|
CASE
|
|
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'before_target_period'
|
|
WHEN first_outcome IS NULL THEN 'unlabeled_first_outcome'
|
|
WHEN prior_episode_count < 1 THEN 'cold_start'
|
|
WHEN prior_total_attempt_count > 50 THEN 'prior_event_count_over_50'
|
|
WHEN coalesce(prior_max_events_in_day, 0) > 4
|
|
THEN 'prior_daily_activity_over_4'
|
|
ELSE NULL
|
|
END AS eligibility_exclusion_reason,
|
|
public_county,
|
|
source_era,
|
|
CASE
|
|
WHEN month(episode_start) IN (12, 1, 2) THEN 'winter'
|
|
WHEN month(episode_start) IN (3, 4, 5) THEN 'spring'
|
|
WHEN month(episode_start) IN (6, 7, 8) THEN 'summer'
|
|
ELSE 'fall'
|
|
END AS target_season,
|
|
CASE
|
|
WHEN last_observed_model_year BETWEEN 1886 AND year(episode_start) + 1
|
|
THEN greatest(year(episode_start) - last_observed_model_year, 0)
|
|
ELSE NULL
|
|
END::INTEGER AS vehicle_age,
|
|
prior_episode_count,
|
|
prior_total_attempt_count::BIGINT AS prior_total_attempt_count,
|
|
prior_attempt_count,
|
|
days_since_prior_episode,
|
|
days_since_prior_adverse,
|
|
prior_nonpass_rate,
|
|
prior_first_outcome,
|
|
prior_final_outcome,
|
|
last_observed_make,
|
|
last_observed_model,
|
|
last_observed_model_year,
|
|
prior_labeled_episode_count,
|
|
prior_pass_count,
|
|
prior_fail_count,
|
|
prior_reject_count,
|
|
prior_abort_count,
|
|
prior_nonpass_count,
|
|
prior_pass_rate,
|
|
last3_labeled_episode_count,
|
|
last3_nonpass_count,
|
|
last3_nonpass_rate,
|
|
prior_max_events_in_day,
|
|
prior_first_outcome IS NULL AS prior_first_outcome_missing,
|
|
last_observed_model_year IS NULL AS prior_model_year_missing,
|
|
(SELECT source_data_kind FROM build_config) AS source_data_kind,
|
|
(SELECT population_estimate_allowed FROM build_config)
|
|
AS population_estimate_allowed
|
|
FROM episode_history;
|
|
|
|
CREATE OR REPLACE TABLE cohort_flow AS
|
|
SELECT
|
|
'event' AS grain,
|
|
'input' AS stage,
|
|
'all_rows' AS reason,
|
|
count(*) AS observation_count,
|
|
count(DISTINCT try_cast(vehicle_token AS VARCHAR)) AS vehicle_count
|
|
FROM stg_events
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
'event',
|
|
'excluded',
|
|
exclusion_reason,
|
|
count(*),
|
|
count(DISTINCT vehicle_token)
|
|
FROM event_exclusions
|
|
GROUP BY exclusion_reason
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
'event',
|
|
'accepted',
|
|
'clean_events',
|
|
count(*),
|
|
count(DISTINCT vehicle_token)
|
|
FROM clean_events
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
'episode',
|
|
'constructed',
|
|
'all_episodes',
|
|
count(*),
|
|
count(DISTINCT vehicle_token)
|
|
FROM inspection_episodes
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
'episode',
|
|
'eligible_returning_target',
|
|
coalesce(eligibility_exclusion_reason, 'eligible'),
|
|
count(*),
|
|
count(DISTINCT vehicle_token)
|
|
FROM feature_mart
|
|
GROUP BY coalesce(eligibility_exclusion_reason, 'eligible');
|