diff --git a/.vscode/tasks.json b/.vscode/tasks.json index f0d73de..79a03c1 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -19,6 +19,15 @@ }, "problemMatcher": [] }, + { + "label": "Countydata: inventory metadata safely", + "type": "shell", + "command": "set -a; source .env; set +a; .venv/bin/python scripts/inventory_metadata.py", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] + }, { "label": "Utah Vehicle Health: run Python tests", "type": "shell", diff --git a/README.md b/README.md index 8e98c4f..bb1c908 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ screenshots, browser JavaScript, or a Bolt project. than workspace settings. 4. Connect to `countydata` and run [sql/00_read_only_connection_check.sql](sql/00_read_only_connection_check.sql). Safe aggregate examples are in [sql/01_safe_data_overview.sql](sql/01_safe_data_overview.sql). +5. To inspect every visible database, relation, column, and a bounded sample of + raw JSON field paths without printing row values, run **Countydata: inventory + metadata safely**. The task uses [scripts/inventory_metadata.py](scripts/inventory_metadata.py) + and enforces the same TLS and read-only requirements. The selected project's first aggregate cohort check is [sql/10_episode_cohort_feasibility.sql](sql/10_episode_cohort_feasibility.sql). diff --git a/docs/data_inventory.md b/docs/data_inventory.md index 063db46..e7aa44e 100644 --- a/docs/data_inventory.md +++ b/docs/data_inventory.md @@ -1,7 +1,32 @@ # Countydata inventory -Inventory date: 2026-07-15. All inspection was performed with read-only -transactions and aggregate queries; no identifiers were exported. +Inventory date: 2026-07-15; live schema revalidated 2026-07-21. All inspection +was performed with read-only transactions, metadata queries, aggregate queries, +and bounded JSON field-path sampling; no identifier values were exported. + +## Live access and full-schema boundary + +The July 21 validation connected successfully from this VS Code workspace using +TLS 1.3 and a session forced into read-only mode. It found 24 visible non-system +relations in `countydata`. The repeatable metadata-only inventory is +`scripts/inventory_metadata.py`, exposed as the VS Code task **Countydata: +inventory metadata safely**. + +The analytical `data` schema contains 10 normalized tables. Four +`fdw_countydata` relations and the `fdw_data.dmv_tax_record*` relations are +source mirrors rather than additional populations. The remaining operational +relations contain API-key names and secrets, application usernames/emails and +password hashes, invitations, sessions, IP addresses, event logs, upload +contents, and ingest state. Their field names were inventoried, but their row +values were not read because they are unrelated to the project and sensitive. + +A bounded raw-JSON field-path sample found vehicle attributes plus VIN and +plate identifiers, station/certificate/calibration fields, technician-license +numbers, odometer, engine/fuel/GVWR/transmission fields, DTC/PID arrays, +readiness monitors, MIL state, and visual inspection results. No owner-name +field was observed in that sample. This does not prove that every historical +source file has the same shape, so raw JSON remains private and out of the +public application. ## Accessible databases diff --git a/scripts/inventory_metadata.py b/scripts/inventory_metadata.py new file mode 100644 index 0000000..7dfa862 --- /dev/null +++ b/scripts/inventory_metadata.py @@ -0,0 +1,149 @@ +"""Inventory accessible PostgreSQL metadata without reading row values. + +This intentionally reports database, schema, relation, and column metadata only. +It never selects application rows or identifier values. +""" + +from __future__ import annotations + +import os +import sys + +import psycopg2 + + +REQUIRED_ENV_VARS = ( + "PGHOST", + "PGPORT", + "PGDATABASE", + "PGUSER", + "PGPASSWORD", + "PGSSLMODE", +) + + +def main() -> int: + missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name)] + if missing: + print("Missing database environment variables: " + ", ".join(missing), file=sys.stderr) + return 1 + + if os.environ["PGSSLMODE"].lower() not in {"require", "verify-ca", "verify-full"}: + print("Refusing to connect because PGSSLMODE does not require TLS.", file=sys.stderr) + return 1 + + connection = psycopg2.connect( + application_name="summer_project_metadata_inventory", + connect_timeout=10, + options="-c default_transaction_read_only=on -c statement_timeout=60000", + ) + connection.set_session(readonly=True) + + try: + with connection, connection.cursor() as cursor: + cursor.execute( + """ + SELECT datname, pg_database_size(datname), has_database_privilege(datname, 'CONNECT') + FROM pg_database + WHERE datallowconn + ORDER BY datname + """ + ) + print("DATABASES") + for name, size_bytes, can_connect in cursor.fetchall(): + print(f"{name}\t{size_bytes}\tconnect={can_connect}") + + cursor.execute( + """ + SELECT + n.nspname, + c.relname, + CASE c.relkind + WHEN 'r' THEN 'table' + WHEN 'p' THEN 'partitioned table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized view' + WHEN 'f' THEN 'foreign table' + ELSE c.relkind::text + END, + COALESCE(s.n_live_tup, c.reltuples)::bigint, + pg_total_relation_size(c.oid) + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid + WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + ORDER BY n.nspname, c.relname + """ + ) + print("RELATIONS") + for schema, relation, kind, estimated_rows, size_bytes in cursor.fetchall(): + print(f"{schema}.{relation}\t{kind}\trows~{estimated_rows}\tbytes={size_bytes}") + + # PostgreSQL's catalogs include metadata for relations where this + # login can see the table but information_schema hides its columns. + cursor.execute( + """ + SELECT + n.nspname, + c.relname, + a.attnum, + a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + NOT a.attnotnull + FROM pg_attribute AS a + JOIN pg_class AS c ON c.oid = a.attrelid + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND a.attnum > 0 + AND NOT a.attisdropped + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + ORDER BY n.nspname, c.relname, a.attnum + """ + ) + print("COLUMNS") + for schema, relation, position, column, data_type, nullable in cursor.fetchall(): + print(f"{schema}.{relation}\t{position}\t{column}\t{data_type}\tnullable={nullable}") + + # A bounded physical-page sample reveals the available JSON field + # names and types without emitting any field values. + print("SAMPLED_JSON_PATHS") + for relation in ("dmv_tax_json", "inspection_json"): + cursor.execute( + f""" + WITH RECURSIVE sampled AS ( + SELECT record + FROM data.{relation} TABLESAMPLE SYSTEM (0.02) + LIMIT 1000 + ), walk(path, value) AS ( + SELECT ARRAY[key], value + FROM sampled + CROSS JOIN LATERAL jsonb_each(record) + WHERE jsonb_typeof(record) = 'object' + UNION ALL + SELECT walk.path || child.key, child.value + FROM walk + CROSS JOIN LATERAL jsonb_each( + CASE + WHEN jsonb_typeof(walk.value) = 'object' THEN walk.value + ELSE '{{}}'::jsonb + END + ) AS child + ) + SELECT array_to_string(path, '.'), jsonb_typeof(value), count(*) + FROM walk + GROUP BY path, jsonb_typeof(value) + ORDER BY path, jsonb_typeof(value) + """ + ) + for path, value_type, sampled_count in cursor.fetchall(): + print(f"data.{relation}\t{path}\t{value_type}\tsampled_count={sampled_count}") + return 0 + finally: + connection.close() + + +if __name__ == "__main__": + raise SystemExit(main())