SummerProject2026/scripts/inventory_metadata.py

"""Inventory accessible PostgreSQL metadata without reading row values.

This intentionally reports database, schema, relation, and column metadata only.
It never selects application rows or identifier values.
"""

from __future__ import annotations

import os
import sys

import psycopg2


REQUIRED_ENV_VARS = (
    "PGHOST",
    "PGPORT",
    "PGDATABASE",
    "PGUSER",
    "PGPASSWORD",
    "PGSSLMODE",
)


def main() -> int:
    missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name)]
    if missing:
        print("Missing database environment variables: " + ", ".join(missing), file=sys.stderr)
        return 1

    if os.environ["PGSSLMODE"].lower() not in {"require", "verify-ca", "verify-full"}:
        print("Refusing to connect because PGSSLMODE does not require TLS.", file=sys.stderr)
        return 1

    connection = psycopg2.connect(
        application_name="summer_project_metadata_inventory",
        connect_timeout=10,
        options="-c default_transaction_read_only=on -c statement_timeout=60000",
    )
    connection.set_session(readonly=True)

    try:
        with connection, connection.cursor() as cursor:
            cursor.execute(
                """
                SELECT datname, pg_database_size(datname), has_database_privilege(datname, 'CONNECT')
                FROM pg_database
                WHERE datallowconn
                ORDER BY datname
                """
            )
            print("DATABASES")
            for name, size_bytes, can_connect in cursor.fetchall():
                print(f"{name}\t{size_bytes}\tconnect={can_connect}")

            cursor.execute(
                """
                SELECT
                    n.nspname,
                    c.relname,
                    CASE c.relkind
                        WHEN 'r' THEN 'table'
                        WHEN 'p' THEN 'partitioned table'
                        WHEN 'v' THEN 'view'
                        WHEN 'm' THEN 'materialized view'
                        WHEN 'f' THEN 'foreign table'
                        ELSE c.relkind::text
                    END,
                    COALESCE(s.n_live_tup, c.reltuples)::bigint,
                    pg_total_relation_size(c.oid)
                FROM pg_class AS c
                JOIN pg_namespace AS n ON n.oid = c.relnamespace
                LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid
                WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
                  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
                  AND n.nspname NOT LIKE 'pg_toast%'
                ORDER BY n.nspname, c.relname
                """
            )
            print("RELATIONS")
            for schema, relation, kind, estimated_rows, size_bytes in cursor.fetchall():
                print(f"{schema}.{relation}\t{kind}\trows~{estimated_rows}\tbytes={size_bytes}")

            # PostgreSQL's catalogs include metadata for relations where this
            # login can see the table but information_schema hides its columns.
            cursor.execute(
                """
                SELECT
                    n.nspname,
                    c.relname,
                    a.attnum,
                    a.attname,
                    pg_catalog.format_type(a.atttypid, a.atttypmod),
                    NOT a.attnotnull
                FROM pg_attribute AS a
                JOIN pg_class AS c ON c.oid = a.attrelid
                JOIN pg_namespace AS n ON n.oid = c.relnamespace
                WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
                  AND a.attnum > 0
                  AND NOT a.attisdropped
                  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
                  AND n.nspname NOT LIKE 'pg_toast%'
                ORDER BY n.nspname, c.relname, a.attnum
                """
            )
            print("COLUMNS")
            for schema, relation, position, column, data_type, nullable in cursor.fetchall():
                print(f"{schema}.{relation}\t{position}\t{column}\t{data_type}\tnullable={nullable}")

            # A bounded physical-page sample reveals the available JSON field
            # names and types without emitting any field values.
            print("SAMPLED_JSON_PATHS")
            for relation in ("dmv_tax_json", "inspection_json"):
                cursor.execute(
                    f"""
                    WITH RECURSIVE sampled AS (
                        SELECT record
                        FROM data.{relation} TABLESAMPLE SYSTEM (0.02)
                        LIMIT 1000
                    ), walk(path, value) AS (
                        SELECT ARRAY[key], value
                        FROM sampled
                        CROSS JOIN LATERAL jsonb_each(record)
                        WHERE jsonb_typeof(record) = 'object'
                        UNION ALL
                        SELECT walk.path || child.key, child.value
                        FROM walk
                        CROSS JOIN LATERAL jsonb_each(
                            CASE
                                WHEN jsonb_typeof(walk.value) = 'object' THEN walk.value
                                ELSE '{{}}'::jsonb
                            END
                        ) AS child
                    )
                    SELECT array_to_string(path, '.'), jsonb_typeof(value), count(*)
                    FROM walk
                    GROUP BY path, jsonb_typeof(value)
                    ORDER BY path, jsonb_typeof(value)
                    """
                )
                for path, value_type, sampled_count in cursor.fetchall():
                    print(f"data.{relation}\t{path}\t{value_type}\tsampled_count={sampled_count}")
        return 0
    finally:
        connection.close()


if __name__ == "__main__":
    raise SystemExit(main())