237 lines
7.0 KiB
SQL
237 lines
7.0 KiB
SQL
-- Aggregate-only feasibility profile for the selected project (label contract v3).
|
|
--
|
|
-- This intentionally samples candidate VINs, keeps their complete histories,
|
|
-- and returns only aggregates. The sample is suitable for cohort development,
|
|
-- not for final population estimates. Run the whole file on countydata.
|
|
BEGIN TRANSACTION READ ONLY;
|
|
|
|
WITH candidate_vins AS MATERIALIZED (
|
|
SELECT DISTINCT vin
|
|
FROM data.inspection_search TABLESAMPLE SYSTEM (0.25) REPEATABLE (20260715)
|
|
WHERE vin IS NOT NULL
|
|
AND length(btrim(vin)) = 17
|
|
),
|
|
sampled_vins AS MATERIALIZED (
|
|
SELECT vin
|
|
FROM candidate_vins
|
|
ORDER BY md5(vin)
|
|
LIMIT 2000
|
|
),
|
|
base AS MATERIALIZED (
|
|
SELECT
|
|
s.vin,
|
|
s.id,
|
|
s.test_start,
|
|
lower(btrim(s.county)) AS source_era,
|
|
CASE
|
|
WHEN lower(btrim(s.county)) IN ('slc', 'slco') THEN 'salt_lake'
|
|
ELSE lower(btrim(s.county))
|
|
END AS public_county,
|
|
CASE upper(btrim(s.overall_result))
|
|
WHEN 'PASS' THEN 'pass'
|
|
WHEN 'P' THEN 'pass'
|
|
WHEN 'FAIL' THEN 'fail'
|
|
WHEN 'F' THEN 'fail'
|
|
WHEN 'REJECT' THEN 'reject'
|
|
WHEN 'ABORT' THEN 'abort'
|
|
ELSE CASE
|
|
WHEN lower(btrim(s.county)) = 'utah'
|
|
AND lower(btrim(s.program_type)) = 'obd'
|
|
AND upper(btrim(s.test_type)) = 'OBD' THEN
|
|
CASE upper(btrim(s.obd_result))
|
|
WHEN 'PASS' THEN 'pass'
|
|
WHEN 'P' THEN 'pass'
|
|
WHEN 'FAIL' THEN 'fail'
|
|
WHEN 'F' THEN 'fail'
|
|
WHEN 'REJECT' THEN 'reject'
|
|
WHEN 'ABORT' THEN 'abort'
|
|
ELSE NULL
|
|
END
|
|
ELSE NULL
|
|
END
|
|
END AS outcome,
|
|
CASE
|
|
WHEN upper(btrim(s.overall_result)) IN (
|
|
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
|
) THEN 'overall_result'
|
|
WHEN lower(btrim(s.county)) = 'utah'
|
|
AND lower(btrim(s.program_type)) = 'obd'
|
|
AND upper(btrim(s.test_type)) = 'OBD'
|
|
AND upper(btrim(s.obd_result)) IN (
|
|
'PASS', 'P', 'FAIL', 'F', 'REJECT', 'ABORT'
|
|
) THEN 'utah_obd_proxy'
|
|
ELSE NULL
|
|
END AS outcome_label_source
|
|
-- Only Utah OBD/OBD rows may use utah_obd_proxy, and only for binary
|
|
-- pass/non-pass analysis. Multiclass analysis must require
|
|
-- outcome_label_source='overall_result'.
|
|
FROM data.inspection_search AS s
|
|
JOIN sampled_vins USING (vin)
|
|
WHERE s.test_start >= timestamp '2010-01-01'
|
|
),
|
|
timestamp_quality AS (
|
|
SELECT
|
|
vin,
|
|
test_start,
|
|
count(
|
|
DISTINCT row(
|
|
source_era,
|
|
public_county,
|
|
coalesce(outcome, '<unlabeled>'),
|
|
coalesce(outcome_label_source, '<unlabeled>')
|
|
)
|
|
) AS analytical_variants_at_timestamp
|
|
FROM base
|
|
GROUP BY 1, 2
|
|
),
|
|
timestamp_groups AS (
|
|
SELECT
|
|
base.*,
|
|
row_number() OVER (
|
|
PARTITION BY vin, test_start
|
|
ORDER BY id
|
|
) AS duplicate_rank
|
|
FROM base
|
|
),
|
|
deduplicated AS (
|
|
SELECT
|
|
g.vin,
|
|
g.id,
|
|
g.test_start,
|
|
g.source_era,
|
|
g.public_county,
|
|
g.outcome,
|
|
g.outcome_label_source
|
|
FROM timestamp_groups AS g
|
|
JOIN timestamp_quality AS q USING (vin, test_start)
|
|
WHERE g.duplicate_rank = 1
|
|
AND q.analytical_variants_at_timestamp = 1
|
|
),
|
|
with_gaps AS (
|
|
SELECT
|
|
*,
|
|
lag(test_start) OVER (
|
|
PARTITION BY vin ORDER BY test_start, id
|
|
) AS previous_test_start
|
|
FROM deduplicated
|
|
),
|
|
episode_markers AS (
|
|
SELECT
|
|
*,
|
|
CASE
|
|
WHEN previous_test_start IS NULL
|
|
OR test_start - previous_test_start > interval '30 days'
|
|
THEN 1 ELSE 0
|
|
END AS starts_episode
|
|
FROM with_gaps
|
|
),
|
|
numbered AS (
|
|
SELECT
|
|
*,
|
|
sum(starts_episode) OVER (
|
|
PARTITION BY vin ORDER BY test_start, id
|
|
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
|
) AS episode_number
|
|
FROM episode_markers
|
|
),
|
|
attempts AS (
|
|
SELECT
|
|
*,
|
|
row_number() OVER (
|
|
PARTITION BY vin, episode_number ORDER BY test_start, id
|
|
) AS attempt_number
|
|
FROM numbered
|
|
),
|
|
episode_summaries AS (
|
|
SELECT
|
|
vin,
|
|
episode_number,
|
|
min(test_start) AS episode_start,
|
|
count(*) AS attempt_count,
|
|
bool_or(outcome = 'pass') AS eventually_passed
|
|
FROM attempts
|
|
GROUP BY 1, 2
|
|
),
|
|
episodes AS (
|
|
SELECT
|
|
a.vin,
|
|
a.episode_number,
|
|
a.test_start AS episode_start,
|
|
a.source_era,
|
|
a.public_county,
|
|
a.outcome AS first_outcome,
|
|
a.outcome_label_source AS first_outcome_label_source,
|
|
s.attempt_count,
|
|
s.eventually_passed
|
|
FROM attempts AS a
|
|
JOIN episode_summaries AS s USING (vin, episode_number)
|
|
WHERE a.attempt_number = 1
|
|
),
|
|
sequenced AS (
|
|
SELECT
|
|
vin,
|
|
episode_number,
|
|
episode_start,
|
|
source_era,
|
|
public_county,
|
|
first_outcome,
|
|
first_outcome_label_source,
|
|
lag(episode_start) OVER (
|
|
PARTITION BY vin ORDER BY episode_number
|
|
) AS prior_episode_start,
|
|
lag(first_outcome) OVER (
|
|
PARTITION BY vin ORDER BY episode_number
|
|
) AS prior_first_outcome,
|
|
lag(attempt_count) OVER (
|
|
PARTITION BY vin ORDER BY episode_number
|
|
) AS prior_attempt_count,
|
|
sum(attempt_count) OVER (
|
|
PARTITION BY vin ORDER BY episode_number
|
|
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
|
) AS prior_event_count
|
|
FROM episodes
|
|
),
|
|
eligible AS (
|
|
SELECT *
|
|
FROM sequenced
|
|
WHERE episode_start >= timestamp '2016-01-01'
|
|
AND episode_number > 1
|
|
AND first_outcome IS NOT NULL
|
|
AND prior_event_count <= 50
|
|
)
|
|
SELECT
|
|
CASE
|
|
WHEN grouping(extract(year FROM episode_start)::integer) = 1
|
|
THEN NULL
|
|
ELSE extract(year FROM episode_start)::integer
|
|
END AS target_year,
|
|
count(*) AS eligible_episodes,
|
|
count(DISTINCT vin) AS vehicles,
|
|
count(*) FILTER (WHERE first_outcome = 'pass') AS pass_episodes,
|
|
count(*) FILTER (
|
|
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
|
) AS nonpass_episodes,
|
|
count(*) FILTER (
|
|
WHERE first_outcome_label_source = 'utah_obd_proxy'
|
|
) AS utah_obd_proxy_episodes,
|
|
round(
|
|
count(*) FILTER (
|
|
WHERE first_outcome IN ('fail', 'reject', 'abort')
|
|
)::numeric / nullif(count(*), 0),
|
|
4
|
|
) AS nonpass_rate,
|
|
percentile_cont(0.5) WITHIN GROUP (
|
|
ORDER BY extract(epoch FROM (episode_start-prior_episode_start))/86400.0
|
|
) AS median_days_since_prior_episode,
|
|
percentile_cont(0.5) WITHIN GROUP (
|
|
ORDER BY prior_attempt_count
|
|
) AS median_prior_attempts
|
|
FROM eligible
|
|
GROUP BY GROUPING SETS (
|
|
(),
|
|
(extract(year FROM episode_start)::integer)
|
|
)
|
|
ORDER BY target_year NULLS FIRST;
|
|
|
|
ROLLBACK;
|