"""Inventory accessible PostgreSQL metadata without reading row values. This intentionally reports database, schema, relation, and column metadata only. It never selects application rows or identifier values. """ from __future__ import annotations import os import sys import psycopg2 REQUIRED_ENV_VARS = ( "PGHOST", "PGPORT", "PGDATABASE", "PGUSER", "PGPASSWORD", "PGSSLMODE", ) def main() -> int: missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name)] if missing: print("Missing database environment variables: " + ", ".join(missing), file=sys.stderr) return 1 if os.environ["PGSSLMODE"].lower() not in {"require", "verify-ca", "verify-full"}: print("Refusing to connect because PGSSLMODE does not require TLS.", file=sys.stderr) return 1 connection = psycopg2.connect( application_name="summer_project_metadata_inventory", connect_timeout=10, options="-c default_transaction_read_only=on -c statement_timeout=60000", ) connection.set_session(readonly=True) try: with connection, connection.cursor() as cursor: cursor.execute( """ SELECT datname, pg_database_size(datname), has_database_privilege(datname, 'CONNECT') FROM pg_database WHERE datallowconn ORDER BY datname """ ) print("DATABASES") for name, size_bytes, can_connect in cursor.fetchall(): print(f"{name}\t{size_bytes}\tconnect={can_connect}") cursor.execute( """ SELECT n.nspname, c.relname, CASE c.relkind WHEN 'r' THEN 'table' WHEN 'p' THEN 'partitioned table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table' ELSE c.relkind::text END, COALESCE(s.n_live_tup, c.reltuples)::bigint, pg_total_relation_size(c.oid) FROM pg_class AS c JOIN pg_namespace AS n ON n.oid = c.relnamespace LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') AND n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname NOT LIKE 'pg_toast%' ORDER BY n.nspname, c.relname """ ) print("RELATIONS") for schema, relation, kind, estimated_rows, size_bytes in cursor.fetchall(): print(f"{schema}.{relation}\t{kind}\trows~{estimated_rows}\tbytes={size_bytes}") # PostgreSQL's catalogs include metadata for relations where this # login can see the table but information_schema hides its columns. cursor.execute( """ SELECT n.nspname, c.relname, a.attnum, a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), NOT a.attnotnull FROM pg_attribute AS a JOIN pg_class AS c ON c.oid = a.attrelid JOIN pg_namespace AS n ON n.oid = c.relnamespace WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') AND a.attnum > 0 AND NOT a.attisdropped AND n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname NOT LIKE 'pg_toast%' ORDER BY n.nspname, c.relname, a.attnum """ ) print("COLUMNS") for schema, relation, position, column, data_type, nullable in cursor.fetchall(): print(f"{schema}.{relation}\t{position}\t{column}\t{data_type}\tnullable={nullable}") # A bounded physical-page sample reveals the available JSON field # names and types without emitting any field values. print("SAMPLED_JSON_PATHS") for relation in ("dmv_tax_json", "inspection_json"): cursor.execute( f""" WITH RECURSIVE sampled AS ( SELECT record FROM data.{relation} TABLESAMPLE SYSTEM (0.02) LIMIT 1000 ), walk(path, value) AS ( SELECT ARRAY[key], value FROM sampled CROSS JOIN LATERAL jsonb_each(record) WHERE jsonb_typeof(record) = 'object' UNION ALL SELECT walk.path || child.key, child.value FROM walk CROSS JOIN LATERAL jsonb_each( CASE WHEN jsonb_typeof(walk.value) = 'object' THEN walk.value ELSE '{{}}'::jsonb END ) AS child ) SELECT array_to_string(path, '.'), jsonb_typeof(value), count(*) FROM walk GROUP BY path, jsonb_typeof(value) ORDER BY path, jsonb_typeof(value) """ ) for path, value_type, sampled_count in cursor.fetchall(): print(f"data.{relation}\t{path}\t{value_type}\tsampled_count={sampled_count}") return 0 finally: connection.close() if __name__ == "__main__": raise SystemExit(main())