initial code
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
-- Build one explicitly point-in-time row per episode. All predictors carrying
|
||||
-- history use a window ending at one episode preceding the target.
|
||||
|
||||
CREATE OR REPLACE TABLE episode_history AS
|
||||
WITH history_windows AS (
|
||||
SELECT
|
||||
e.*,
|
||||
count(*) OVER prior_all AS prior_episode_count,
|
||||
coalesce(sum(attempt_count) OVER prior_all, 0) AS prior_total_attempt_count,
|
||||
count(first_outcome) OVER prior_all AS prior_labeled_episode_count,
|
||||
lag(attempt_count) OVER by_vehicle AS prior_attempt_count,
|
||||
lag(first_outcome) OVER by_vehicle AS prior_first_outcome,
|
||||
lag(final_outcome) OVER by_vehicle AS prior_final_outcome,
|
||||
lag(episode_start) OVER by_vehicle AS prior_episode_start,
|
||||
max(CASE
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort')
|
||||
THEN episode_start
|
||||
END) OVER prior_all AS prior_adverse_episode_start,
|
||||
count(*) FILTER (WHERE first_outcome = 'pass') OVER prior_all
|
||||
AS prior_pass_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'fail') OVER prior_all
|
||||
AS prior_fail_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'reject') OVER prior_all
|
||||
AS prior_reject_count,
|
||||
count(*) FILTER (WHERE first_outcome = 'abort') OVER prior_all
|
||||
AS prior_abort_count,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) OVER prior_all AS prior_nonpass_count,
|
||||
count(first_outcome) OVER prior_three AS last3_labeled_episode_count,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) OVER prior_three AS last3_nonpass_count,
|
||||
max(max_events_in_day) OVER prior_all AS prior_max_events_in_day,
|
||||
arg_max(episode_last_observed_make, episode_number)
|
||||
FILTER (WHERE episode_last_observed_make IS NOT NULL)
|
||||
OVER prior_all AS last_observed_make,
|
||||
arg_max(episode_last_observed_model, episode_number)
|
||||
FILTER (WHERE episode_last_observed_model IS NOT NULL)
|
||||
OVER prior_all AS last_observed_model,
|
||||
arg_max(episode_last_observed_model_year, episode_number)
|
||||
FILTER (WHERE episode_last_observed_model_year IS NOT NULL)
|
||||
OVER prior_all AS last_observed_model_year
|
||||
FROM inspection_episodes AS e
|
||||
WINDOW
|
||||
by_vehicle AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
),
|
||||
prior_all AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
),
|
||||
prior_three AS (
|
||||
PARTITION BY vehicle_token
|
||||
ORDER BY episode_number
|
||||
ROWS BETWEEN 3 PRECEDING AND 1 PRECEDING
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN prior_episode_start IS NULL THEN NULL
|
||||
ELSE extract(epoch FROM (episode_start - prior_episode_start)) / 86400.0
|
||||
END AS days_since_prior_episode,
|
||||
CASE
|
||||
WHEN prior_adverse_episode_start IS NULL THEN NULL
|
||||
ELSE extract(epoch FROM (episode_start - prior_adverse_episode_start))
|
||||
/ 86400.0
|
||||
END AS days_since_prior_adverse,
|
||||
prior_nonpass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
||||
AS prior_nonpass_rate,
|
||||
prior_pass_count::DOUBLE / nullif(prior_labeled_episode_count, 0)
|
||||
AS prior_pass_rate,
|
||||
last3_nonpass_count::DOUBLE / nullif(last3_labeled_episode_count, 0)
|
||||
AS last3_nonpass_rate
|
||||
FROM history_windows;
|
||||
|
||||
CREATE OR REPLACE TABLE feature_mart AS
|
||||
SELECT
|
||||
vehicle_token,
|
||||
vehicle_bucket,
|
||||
vehicle_bucket < 10 AS is_vin_audit,
|
||||
episode_number::BIGINT AS episode_number,
|
||||
episode_start,
|
||||
first_outcome,
|
||||
first_outcome_label_source AS target_outcome_label_source,
|
||||
CASE
|
||||
WHEN first_outcome = 'pass' THEN 0
|
||||
WHEN first_outcome IN ('fail', 'reject', 'abort') THEN 1
|
||||
ELSE NULL
|
||||
END::INTEGER AS target_nonpass,
|
||||
episode_start >= TIMESTAMP '2016-01-01'
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_episode_count >= 1
|
||||
AND prior_total_attempt_count <= 50
|
||||
AND coalesce(prior_max_events_in_day, 0) <= 4
|
||||
AS eligible_returning_target,
|
||||
CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'historical_context'
|
||||
WHEN episode_start < TIMESTAMP '2023-01-01' THEN 'train'
|
||||
WHEN episode_start < TIMESTAMP '2024-01-01' THEN 'tune'
|
||||
WHEN episode_start < TIMESTAMP '2025-01-01' THEN 'calibrate'
|
||||
WHEN episode_start < TIMESTAMP '2026-01-01' THEN 'test'
|
||||
WHEN episode_start < TIMESTAMP '2027-01-01' THEN 'shadow'
|
||||
ELSE 'out_of_scope'
|
||||
END AS temporal_partition,
|
||||
CASE
|
||||
WHEN episode_start < TIMESTAMP '2016-01-01' THEN 'before_target_period'
|
||||
WHEN first_outcome IS NULL THEN 'unlabeled_first_outcome'
|
||||
WHEN prior_episode_count < 1 THEN 'cold_start'
|
||||
WHEN prior_total_attempt_count > 50 THEN 'prior_event_count_over_50'
|
||||
WHEN coalesce(prior_max_events_in_day, 0) > 4
|
||||
THEN 'prior_daily_activity_over_4'
|
||||
ELSE NULL
|
||||
END AS eligibility_exclusion_reason,
|
||||
public_county,
|
||||
source_era,
|
||||
CASE
|
||||
WHEN month(episode_start) IN (12, 1, 2) THEN 'winter'
|
||||
WHEN month(episode_start) IN (3, 4, 5) THEN 'spring'
|
||||
WHEN month(episode_start) IN (6, 7, 8) THEN 'summer'
|
||||
ELSE 'fall'
|
||||
END AS target_season,
|
||||
CASE
|
||||
WHEN last_observed_model_year BETWEEN 1886 AND year(episode_start) + 1
|
||||
THEN greatest(year(episode_start) - last_observed_model_year, 0)
|
||||
ELSE NULL
|
||||
END::INTEGER AS vehicle_age,
|
||||
prior_episode_count,
|
||||
prior_total_attempt_count::BIGINT AS prior_total_attempt_count,
|
||||
prior_attempt_count,
|
||||
days_since_prior_episode,
|
||||
days_since_prior_adverse,
|
||||
prior_nonpass_rate,
|
||||
prior_first_outcome,
|
||||
prior_final_outcome,
|
||||
last_observed_make,
|
||||
last_observed_model,
|
||||
last_observed_model_year,
|
||||
prior_labeled_episode_count,
|
||||
prior_pass_count,
|
||||
prior_fail_count,
|
||||
prior_reject_count,
|
||||
prior_abort_count,
|
||||
prior_nonpass_count,
|
||||
prior_pass_rate,
|
||||
last3_labeled_episode_count,
|
||||
last3_nonpass_count,
|
||||
last3_nonpass_rate,
|
||||
prior_max_events_in_day,
|
||||
prior_first_outcome IS NULL AS prior_first_outcome_missing,
|
||||
last_observed_model_year IS NULL AS prior_model_year_missing,
|
||||
(SELECT source_data_kind FROM build_config) AS source_data_kind,
|
||||
(SELECT population_estimate_allowed FROM build_config)
|
||||
AS population_estimate_allowed
|
||||
FROM episode_history;
|
||||
|
||||
CREATE OR REPLACE TABLE cohort_flow AS
|
||||
SELECT
|
||||
'event' AS grain,
|
||||
'input' AS stage,
|
||||
'all_rows' AS reason,
|
||||
count(*) AS observation_count,
|
||||
count(DISTINCT try_cast(vehicle_token AS VARCHAR)) AS vehicle_count
|
||||
FROM stg_events
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'event',
|
||||
'excluded',
|
||||
exclusion_reason,
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM event_exclusions
|
||||
GROUP BY exclusion_reason
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'event',
|
||||
'accepted',
|
||||
'clean_events',
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM clean_events
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode',
|
||||
'constructed',
|
||||
'all_episodes',
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM inspection_episodes
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
'episode',
|
||||
'eligible_returning_target',
|
||||
coalesce(eligibility_exclusion_reason, 'eligible'),
|
||||
count(*),
|
||||
count(DISTINCT vehicle_token)
|
||||
FROM feature_mart
|
||||
GROUP BY coalesce(eligibility_exclusion_reason, 'eligible');
|
||||
Reference in New Issue
Block a user