initial code
This commit is contained in:
@@ -0,0 +1,236 @@
|
||||
-- Aggregate-only feasibility profile for the selected project (label contract v3).
|
||||
--
|
||||
-- This intentionally samples candidate VINs, keeps their complete histories,
|
||||
-- and returns only aggregates. The sample is suitable for cohort development,
|
||||
-- not for final population estimates. Run the whole file on countydata.
|
||||
BEGIN TRANSACTION READ ONLY;
|
||||
|
||||
WITH candidate_vins AS MATERIALIZED (
|
||||
SELECT DISTINCT vin
|
||||
FROM data.inspection_search TABLESAMPLE SYSTEM (0.25) REPEATABLE (20260715)
|
||||
WHERE vin IS NOT NULL
|
||||
AND length(btrim(vin)) = 17
|
||||
),
|
||||
sampled_vins AS MATERIALIZED (
|
||||
SELECT vin
|
||||
FROM candidate_vins
|
||||
ORDER BY md5(vin)
|
||||
LIMIT 2000
|
||||
),
|
||||
base AS MATERIALIZED (
|
||||
SELECT
|
||||
s.vin,
|
||||
s.id,
|
||||
s.test_start,
|
||||
lower(btrim(s.county)) AS source_era,
|
||||
CASE
|
||||
WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake'
|
||||
ELSE lower(btrim(s.county))
|
||||
END AS public_county,
|
||||
CASE upper(btrim(s.overall_result))
|
||||
WHEN 'PASS' THEN 'pass'
|
||||
WHEN 'P' THEN 'pass'
|
||||
WHEN 'FAIL' THEN 'fail'
|
||||
WHEN 'F' THEN 'fail'
|
||||
WHEN 'REJECT' THEN 'reject'
|
||||
WHEN 'ABORT' THEN 'abort'
|
||||
ELSE CASE
|
||||
WHEN lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD' THEN
|
||||
CASE upper(btrim(s.obd_result))
|
||||
WHEN 'PASS' THEN 'pass'
|
||||
WHEN 'P' THEN 'pass'
|
||||
WHEN 'FAIL' THEN 'fail'
|
||||
WHEN 'F' THEN 'fail'
|
||||
WHEN 'REJECT' THEN 'reject'
|
||||
WHEN 'ABORT' THEN 'abort'
|
||||
ELSE NULL
|
||||
END
|
||||
ELSE NULL
|
||||
END
|
||||
END AS outcome,
|
||||
CASE
|
||||
WHEN upper(btrim(s.overall_result)) IN (
|
||||
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
||||
) THEN 'overall_result'
|
||||
WHEN lower(btrim(s.county)) = 'utah'
|
||||
AND lower(btrim(s.program_type)) = 'obd'
|
||||
AND upper(btrim(s.test_type)) = 'OBD'
|
||||
AND upper(btrim(s.obd_result)) IN (
|
||||
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
||||
) THEN 'utah_obd_proxy'
|
||||
ELSE NULL
|
||||
END AS outcome_label_source
|
||||
-- Only Utah OBD/OBD rows may use utah_obd_proxy, and only for binary
|
||||
-- pass/non-pass analysis. Multiclass analysis must require
|
||||
-- outcome_label_source='overall_result'.
|
||||
FROM data.inspection_search AS s
|
||||
JOIN sampled_vins USING (vin)
|
||||
WHERE s.test_start >= timestamp '2010-01-01'
|
||||
),
|
||||
timestamp_quality AS (
|
||||
SELECT
|
||||
vin,
|
||||
test_start,
|
||||
count(
|
||||
DISTINCT row(
|
||||
source_era,
|
||||
public_county,
|
||||
coalesce(outcome, '<unlabeled>'),
|
||||
coalesce(outcome_label_source, '<unlabeled>')
|
||||
)
|
||||
) AS analytical_variants_at_timestamp
|
||||
FROM base
|
||||
GROUP BY 1, 2
|
||||
),
|
||||
timestamp_groups AS (
|
||||
SELECT
|
||||
base.*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vin, test_start
|
||||
ORDER BY id
|
||||
) AS duplicate_rank
|
||||
FROM base
|
||||
),
|
||||
deduplicated AS (
|
||||
SELECT
|
||||
g.vin,
|
||||
g.id,
|
||||
g.test_start,
|
||||
g.source_era,
|
||||
g.public_county,
|
||||
g.outcome,
|
||||
g.outcome_label_source
|
||||
FROM timestamp_groups AS g
|
||||
JOIN timestamp_quality AS q USING (vin, test_start)
|
||||
WHERE g.duplicate_rank = 1
|
||||
AND q.analytical_variants_at_timestamp = 1
|
||||
),
|
||||
with_gaps AS (
|
||||
SELECT
|
||||
*,
|
||||
lag(test_start) OVER (
|
||||
PARTITION BY vin ORDER BY test_start, id
|
||||
) AS previous_test_start
|
||||
FROM deduplicated
|
||||
),
|
||||
episode_markers AS (
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN previous_test_start IS NULL
|
||||
OR test_start - previous_test_start > interval '30 days'
|
||||
THEN 1 ELSE 0
|
||||
END AS starts_episode
|
||||
FROM with_gaps
|
||||
),
|
||||
numbered AS (
|
||||
SELECT
|
||||
*,
|
||||
sum(starts_episode) OVER (
|
||||
PARTITION BY vin ORDER BY test_start, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS episode_number
|
||||
FROM episode_markers
|
||||
),
|
||||
attempts AS (
|
||||
SELECT
|
||||
*,
|
||||
row_number() OVER (
|
||||
PARTITION BY vin, episode_number ORDER BY test_start, id
|
||||
) AS attempt_number
|
||||
FROM numbered
|
||||
),
|
||||
episode_summaries AS (
|
||||
SELECT
|
||||
vin,
|
||||
episode_number,
|
||||
min(test_start) AS episode_start,
|
||||
count(*) AS attempt_count,
|
||||
bool_or(outcome = 'pass') AS eventually_passed
|
||||
FROM attempts
|
||||
GROUP BY 1, 2
|
||||
),
|
||||
episodes AS (
|
||||
SELECT
|
||||
a.vin,
|
||||
a.episode_number,
|
||||
a.test_start AS episode_start,
|
||||
a.source_era,
|
||||
a.public_county,
|
||||
a.outcome AS first_outcome,
|
||||
a.outcome_label_source AS first_outcome_label_source,
|
||||
s.attempt_count,
|
||||
s.eventually_passed
|
||||
FROM attempts AS a
|
||||
JOIN episode_summaries AS s USING (vin, episode_number)
|
||||
WHERE a.attempt_number = 1
|
||||
),
|
||||
sequenced AS (
|
||||
SELECT
|
||||
vin,
|
||||
episode_number,
|
||||
episode_start,
|
||||
source_era,
|
||||
public_county,
|
||||
first_outcome,
|
||||
first_outcome_label_source,
|
||||
lag(episode_start) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_episode_start,
|
||||
lag(first_outcome) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_first_outcome,
|
||||
lag(attempt_count) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
) AS prior_attempt_count,
|
||||
sum(attempt_count) OVER (
|
||||
PARTITION BY vin ORDER BY episode_number
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
) AS prior_event_count
|
||||
FROM episodes
|
||||
),
|
||||
eligible AS (
|
||||
SELECT *
|
||||
FROM sequenced
|
||||
WHERE episode_start >= timestamp '2016-01-01'
|
||||
AND episode_number > 1
|
||||
AND first_outcome IS NOT NULL
|
||||
AND prior_event_count <= 50
|
||||
)
|
||||
SELECT
|
||||
CASE
|
||||
WHEN grouping(extract(year FROM episode_start)::integer) = 1
|
||||
THEN NULL
|
||||
ELSE extract(year FROM episode_start)::integer
|
||||
END AS target_year,
|
||||
count(*) AS eligible_episodes,
|
||||
count(DISTINCT vin) AS vehicles,
|
||||
count(*) FILTER (WHERE first_outcome = 'pass') AS pass_episodes,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
) AS nonpass_episodes,
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome_label_source = 'utah_obd_proxy'
|
||||
) AS utah_obd_proxy_episodes,
|
||||
round(
|
||||
count(*) FILTER (
|
||||
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
||||
)::numeric / nullif(count(*), 0),
|
||||
4
|
||||
) AS nonpass_rate,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY extract(epoch FROM (episode_start-prior_episode_start))/86400.0
|
||||
) AS median_days_since_prior_episode,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY prior_attempt_count
|
||||
) AS median_prior_attempts
|
||||
FROM eligible
|
||||
GROUP BY GROUPING SETS (
|
||||
(),
|
||||
(extract(year FROM episode_start)::integer)
|
||||
)
|
||||
ORDER BY target_year NULLS FIRST;
|
||||
|
||||
ROLLBACK;
|
||||
Reference in New Issue
Block a user