data and metadata inventory stuff
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
"""Inventory accessible PostgreSQL metadata without reading row values.
|
||||
|
||||
This intentionally reports database, schema, relation, and column metadata only.
|
||||
It never selects application rows or identifier values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
|
||||
REQUIRED_ENV_VARS = (
|
||||
"PGHOST",
|
||||
"PGPORT",
|
||||
"PGDATABASE",
|
||||
"PGUSER",
|
||||
"PGPASSWORD",
|
||||
"PGSSLMODE",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name)]
|
||||
if missing:
|
||||
print("Missing database environment variables: " + ", ".join(missing), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if os.environ["PGSSLMODE"].lower() not in {"require", "verify-ca", "verify-full"}:
|
||||
print("Refusing to connect because PGSSLMODE does not require TLS.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
connection = psycopg2.connect(
|
||||
application_name="summer_project_metadata_inventory",
|
||||
connect_timeout=10,
|
||||
options="-c default_transaction_read_only=on -c statement_timeout=60000",
|
||||
)
|
||||
connection.set_session(readonly=True)
|
||||
|
||||
try:
|
||||
with connection, connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT datname, pg_database_size(datname), has_database_privilege(datname, 'CONNECT')
|
||||
FROM pg_database
|
||||
WHERE datallowconn
|
||||
ORDER BY datname
|
||||
"""
|
||||
)
|
||||
print("DATABASES")
|
||||
for name, size_bytes, can_connect in cursor.fetchall():
|
||||
print(f"{name}\t{size_bytes}\tconnect={can_connect}")
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
n.nspname,
|
||||
c.relname,
|
||||
CASE c.relkind
|
||||
WHEN 'r' THEN 'table'
|
||||
WHEN 'p' THEN 'partitioned table'
|
||||
WHEN 'v' THEN 'view'
|
||||
WHEN 'm' THEN 'materialized view'
|
||||
WHEN 'f' THEN 'foreign table'
|
||||
ELSE c.relkind::text
|
||||
END,
|
||||
COALESCE(s.n_live_tup, c.reltuples)::bigint,
|
||||
pg_total_relation_size(c.oid)
|
||||
FROM pg_class AS c
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_stat_user_tables AS s ON s.relid = c.oid
|
||||
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_toast%'
|
||||
ORDER BY n.nspname, c.relname
|
||||
"""
|
||||
)
|
||||
print("RELATIONS")
|
||||
for schema, relation, kind, estimated_rows, size_bytes in cursor.fetchall():
|
||||
print(f"{schema}.{relation}\t{kind}\trows~{estimated_rows}\tbytes={size_bytes}")
|
||||
|
||||
# PostgreSQL's catalogs include metadata for relations where this
|
||||
# login can see the table but information_schema hides its columns.
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
n.nspname,
|
||||
c.relname,
|
||||
a.attnum,
|
||||
a.attname,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod),
|
||||
NOT a.attnotnull
|
||||
FROM pg_attribute AS a
|
||||
JOIN pg_class AS c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_toast%'
|
||||
ORDER BY n.nspname, c.relname, a.attnum
|
||||
"""
|
||||
)
|
||||
print("COLUMNS")
|
||||
for schema, relation, position, column, data_type, nullable in cursor.fetchall():
|
||||
print(f"{schema}.{relation}\t{position}\t{column}\t{data_type}\tnullable={nullable}")
|
||||
|
||||
# A bounded physical-page sample reveals the available JSON field
|
||||
# names and types without emitting any field values.
|
||||
print("SAMPLED_JSON_PATHS")
|
||||
for relation in ("dmv_tax_json", "inspection_json"):
|
||||
cursor.execute(
|
||||
f"""
|
||||
WITH RECURSIVE sampled AS (
|
||||
SELECT record
|
||||
FROM data.{relation} TABLESAMPLE SYSTEM (0.02)
|
||||
LIMIT 1000
|
||||
), walk(path, value) AS (
|
||||
SELECT ARRAY[key], value
|
||||
FROM sampled
|
||||
CROSS JOIN LATERAL jsonb_each(record)
|
||||
WHERE jsonb_typeof(record) = 'object'
|
||||
UNION ALL
|
||||
SELECT walk.path || child.key, child.value
|
||||
FROM walk
|
||||
CROSS JOIN LATERAL jsonb_each(
|
||||
CASE
|
||||
WHEN jsonb_typeof(walk.value) = 'object' THEN walk.value
|
||||
ELSE '{{}}'::jsonb
|
||||
END
|
||||
) AS child
|
||||
)
|
||||
SELECT array_to_string(path, '.'), jsonb_typeof(value), count(*)
|
||||
FROM walk
|
||||
GROUP BY path, jsonb_typeof(value)
|
||||
ORDER BY path, jsonb_typeof(value)
|
||||
"""
|
||||
)
|
||||
for path, value_type, sampled_count in cursor.fetchall():
|
||||
print(f"data.{relation}\t{path}\t{value_type}\tsampled_count={sampled_count}")
|
||||
return 0
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user