334 lines
15 KiB
Python
334 lines
15 KiB
Python
"""Predefined, read-only Firebird catalog metadata collection.
|
|
|
|
Mission 00D: the concrete reader consumed here is
|
|
``gti_discovery.catalog.FirebirdCatalog`` -- a closed API with no
|
|
``execute(sql)``-shaped method at all. ``collect_metadata`` only ever
|
|
calls the narrow, named operations declared on ``ReadOnlyCatalogReader``
|
|
below; it has no path to send arbitrary SQL even if a caller wanted it
|
|
to. The private immutable catalog operation map contains fixed,
|
|
project-owned strings that ``FirebirdCatalog`` alone executes -- they never depend on the offline
|
|
lexical checker in ``gti_discovery.read_only``, which is no longer part
|
|
of the runtime path (see that module's docstring).
|
|
"""
|
|
|
|
from collections.abc import Mapping
|
|
import json
|
|
from types import MappingProxyType
|
|
from typing import Any
|
|
|
|
|
|
_CATALOG_QUERIES = MappingProxyType({
|
|
"database": "SELECT MON$DATABASE_NAME, MON$ODS_MAJOR, MON$ODS_MINOR FROM MON$DATABASE",
|
|
"relations": "SELECT RDB$RELATION_NAME AS RELATION_NAME, RDB$RELATION_ID AS RELATION_ID, RDB$SYSTEM_FLAG AS SYSTEM_FLAG FROM RDB$RELATIONS",
|
|
"fields": "SELECT RF.RDB$RELATION_NAME AS RELATION_NAME, RF.RDB$FIELD_NAME AS FIELD_NAME, RF.RDB$FIELD_POSITION AS FIELD_POSITION, RF.RDB$NULL_FLAG AS NULL_FLAG, F.RDB$FIELD_TYPE AS FIELD_TYPE, F.RDB$FIELD_SUB_TYPE AS FIELD_SUB_TYPE, F.RDB$FIELD_LENGTH AS FIELD_LENGTH, F.RDB$FIELD_PRECISION AS FIELD_PRECISION, F.RDB$FIELD_SCALE AS FIELD_SCALE, T.RDB$TYPE_NAME AS TYPE_NAME FROM RDB$RELATION_FIELDS RF JOIN RDB$FIELDS F ON F.RDB$FIELD_NAME = RF.RDB$FIELD_SOURCE LEFT JOIN RDB$TYPES T ON T.RDB$TYPE = F.RDB$FIELD_TYPE AND T.RDB$FIELD_NAME = 'RDB$FIELD_TYPE'",
|
|
"keys": "SELECT RC.RDB$CONSTRAINT_NAME AS CONSTRAINT_NAME, RC.RDB$RELATION_NAME AS RELATION_NAME, RC.RDB$CONSTRAINT_TYPE AS CONSTRAINT_TYPE, RC.RDB$INDEX_NAME AS INDEX_NAME, S.RDB$FIELD_NAME AS FIELD_NAME, S.RDB$FIELD_POSITION AS FIELD_POSITION FROM RDB$RELATION_CONSTRAINTS RC LEFT JOIN RDB$INDEX_SEGMENTS S ON S.RDB$INDEX_NAME = RC.RDB$INDEX_NAME",
|
|
"indexes": "SELECT I.RDB$INDEX_NAME AS INDEX_NAME, I.RDB$RELATION_NAME AS RELATION_NAME, I.RDB$UNIQUE_FLAG AS UNIQUE_FLAG, I.RDB$INDEX_TYPE AS INDEX_TYPE, S.RDB$FIELD_NAME AS FIELD_NAME, S.RDB$FIELD_POSITION AS FIELD_POSITION FROM RDB$INDICES I LEFT JOIN RDB$INDEX_SEGMENTS S ON S.RDB$INDEX_NAME = I.RDB$INDEX_NAME",
|
|
"foreign_keys": "SELECT C.RDB$CONSTRAINT_NAME AS CONSTRAINT_NAME, C.RDB$RELATION_NAME AS CHILD_RELATION_NAME, C.RDB$INDEX_NAME AS CHILD_INDEX_NAME, CS.RDB$FIELD_NAME AS CHILD_FIELD_NAME, CS.RDB$FIELD_POSITION AS CHILD_FIELD_POSITION, U.RDB$RELATION_NAME AS REFERENCED_RELATION_NAME, U.RDB$INDEX_NAME AS REFERENCED_INDEX_NAME, US.RDB$FIELD_NAME AS REFERENCED_FIELD_NAME, US.RDB$FIELD_POSITION AS REFERENCED_FIELD_POSITION, R.RDB$UPDATE_RULE AS UPDATE_RULE, R.RDB$DELETE_RULE AS DELETE_RULE FROM RDB$REF_CONSTRAINTS R JOIN RDB$RELATION_CONSTRAINTS C ON C.RDB$CONSTRAINT_NAME = R.RDB$CONSTRAINT_NAME JOIN RDB$RELATION_CONSTRAINTS U ON U.RDB$CONSTRAINT_NAME = R.RDB$CONST_NAME_UQ LEFT JOIN RDB$INDEX_SEGMENTS CS ON CS.RDB$INDEX_NAME = C.RDB$INDEX_NAME LEFT JOIN RDB$INDEX_SEGMENTS US ON US.RDB$INDEX_NAME = U.RDB$INDEX_NAME AND US.RDB$FIELD_POSITION = CS.RDB$FIELD_POSITION",
|
|
"triggers": "SELECT RDB$TRIGGER_NAME AS TRIGGER_NAME, RDB$RELATION_NAME AS RELATION_NAME, RDB$TRIGGER_TYPE AS TRIGGER_TYPE, RDB$TRIGGER_SEQUENCE AS TRIGGER_SEQUENCE, RDB$TRIGGER_INACTIVE AS TRIGGER_INACTIVE FROM RDB$TRIGGERS",
|
|
"procedures": "SELECT RDB$PROCEDURE_NAME AS PROCEDURE_NAME, RDB$PROCEDURE_ID AS PROCEDURE_ID FROM RDB$PROCEDURES",
|
|
"procedure_parameters": "SELECT RDB$PROCEDURE_NAME AS PROCEDURE_NAME, RDB$PARAMETER_NAME AS PARAMETER_NAME, RDB$PARAMETER_NUMBER AS PARAMETER_NUMBER, RDB$PARAMETER_TYPE AS PARAMETER_TYPE, RDB$FIELD_TYPE AS FIELD_TYPE, RDB$FIELD_SUB_TYPE AS FIELD_SUB_TYPE, RDB$FIELD_LENGTH AS FIELD_LENGTH, RDB$FIELD_PRECISION AS FIELD_PRECISION, RDB$FIELD_SCALE AS FIELD_SCALE FROM RDB$PROCEDURE_PARAMETERS",
|
|
"packages": "SELECT RDB$PACKAGE_NAME AS PACKAGE_NAME, RDB$PACKAGE_ID AS PACKAGE_ID FROM RDB$PACKAGES",
|
|
"generators": "SELECT RDB$GENERATOR_NAME AS GENERATOR_NAME, RDB$GENERATOR_ID AS GENERATOR_ID, RDB$SYSTEM_FLAG AS SYSTEM_FLAG FROM RDB$GENERATORS",
|
|
"dependencies": "SELECT RDB$DEPENDENT_NAME AS DEPENDENT_NAME, RDB$DEPENDED_ON_NAME AS DEPENDED_ON_NAME, RDB$DEPENDENT_TYPE AS DEPENDENT_TYPE, RDB$DEPENDED_ON_TYPE AS DEPENDED_ON_TYPE, RDB$FIELD_NAME AS FIELD_NAME FROM RDB$DEPENDENCIES",
|
|
"monitoring": "SELECT MON$ATTACHMENT_ID AS ATTACHMENT_ID, MON$SERVER_PID AS SERVER_PID, MON$STATE AS STATE, MON$TIMESTAMP AS TIMESTAMP FROM MON$ATTACHMENTS",
|
|
})
|
|
|
|
_SENSITIVE_WORDS = (
|
|
"PASSWORD",
|
|
"SECRET",
|
|
"TOKEN",
|
|
"CREDENTIAL",
|
|
"AUTH",
|
|
"BLOB",
|
|
"SOURCE",
|
|
"DEFAULT",
|
|
"BODY",
|
|
)
|
|
_MAX_SAFE_STRING = 1024
|
|
_SAMPLE_SENSITIVE_WORDS = _SENSITIVE_WORDS + (
|
|
"CUSTOMER", "CLIENT", "PRICE", "AMOUNT", "COST", "EMAIL", "PHONE",
|
|
"ADDRESS", "TAX", "CPF", "CNPJ",
|
|
)
|
|
_FIXTURE_REGISTRATION_TOKEN = object()
|
|
_APPROVED_FIXTURE_TYPES = MappingProxyType({})
|
|
|
|
|
|
def _approved_fixture_reader(reader_type: type | None = None, *, token: object) -> type:
|
|
"""Register one exact fixture type through the private test capability."""
|
|
if token is not _FIXTURE_REGISTRATION_TOKEN:
|
|
raise TypeError("fixture registration is private")
|
|
def register(candidate: type) -> type:
|
|
global _APPROVED_FIXTURE_TYPES
|
|
if not isinstance(candidate, type):
|
|
raise TypeError("fixture registration requires a type")
|
|
_APPROVED_FIXTURE_TYPES = MappingProxyType(
|
|
{**_APPROVED_FIXTURE_TYPES, candidate: True}
|
|
)
|
|
return candidate
|
|
return register(reader_type) if reader_type is not None else register
|
|
|
|
|
|
class ReadOnlyCatalogReader:
|
|
"""Approved structured reader contract used by the collector."""
|
|
|
|
__slots__ = ()
|
|
|
|
def get_database_info(self):
|
|
return self.list_database()
|
|
|
|
def list_database(self):
|
|
raise NotImplementedError
|
|
|
|
def list_relations(self):
|
|
raise NotImplementedError
|
|
|
|
def get_relation_columns(self, relation):
|
|
return [row for row in self.list_relation_fields() if row.get("RELATION_NAME", "").strip().upper() == str(relation).strip().upper()]
|
|
|
|
def get_primary_keys(self, relation):
|
|
return [row for row in self.list_keys() if row.get("RELATION_NAME", "").strip().upper() == str(relation).strip().upper()]
|
|
|
|
def get_foreign_keys(self, relation):
|
|
known = str(relation).strip().upper()
|
|
return [row for row in self.list_foreign_keys() if known in {
|
|
str(row.get("CHILD_RELATION_NAME", "")).strip().upper(),
|
|
str(row.get("REFERENCED_RELATION_NAME", "")).strip().upper(),
|
|
}]
|
|
|
|
def get_indices(self, relation):
|
|
return [row for row in self.list_indexes() if row.get("RELATION_NAME", "").strip().upper() == str(relation).strip().upper()]
|
|
|
|
def get_triggers(self):
|
|
return self.list_triggers()
|
|
|
|
def get_procedures(self):
|
|
return self.list_procedures()
|
|
|
|
def get_procedure_parameters(self):
|
|
return self.list_procedure_parameters()
|
|
|
|
def get_packages(self):
|
|
return self.list_packages()
|
|
|
|
def get_generators(self):
|
|
return self.list_generators()
|
|
|
|
def get_dependencies(self):
|
|
return self.list_dependencies()
|
|
|
|
def get_monitoring(self):
|
|
return self.list_monitoring()
|
|
|
|
|
|
def _safe_value(value: Any) -> Any:
|
|
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
return "<redacted>"
|
|
if isinstance(value, Mapping):
|
|
return {
|
|
str(key): _safe_value(item)
|
|
for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))
|
|
if not any(word in str(key).upper() for word in _SENSITIVE_WORDS)
|
|
}
|
|
if isinstance(value, (list, tuple)):
|
|
return [_safe_value(item) for item in value]
|
|
if isinstance(value, str):
|
|
upper_value = value.upper()
|
|
if any(word in upper_value for word in _SENSITIVE_WORDS):
|
|
return "<redacted>"
|
|
return value if len(value) <= _MAX_SAFE_STRING else "<redacted>"
|
|
if isinstance(value, (int, float, bool)) or value is None:
|
|
return value
|
|
return "<redacted>"
|
|
|
|
|
|
def _safe_sample_rows(rows: Any) -> list[dict[str, Any]]:
|
|
"""Keep sampled evidence flat, scalar, bounded, and mapping-relevant."""
|
|
sanitized = []
|
|
for row in rows if isinstance(rows, list) else []:
|
|
if not isinstance(row, Mapping):
|
|
continue
|
|
safe_row = {}
|
|
for key, value in row.items():
|
|
name = str(key)
|
|
upper_name = name.upper()
|
|
if "BLOB" in upper_name:
|
|
safe_row[name] = "<redacted>"
|
|
continue
|
|
if any(word in upper_name for word in _SAMPLE_SENSITIVE_WORDS):
|
|
continue
|
|
if isinstance(value, (str, int, float, bool)) or value is None:
|
|
safe_row[name] = _safe_value(value)
|
|
sanitized.append(safe_row)
|
|
return sanitized
|
|
|
|
|
|
def _row_sort_key(row: Any) -> str:
|
|
return json.dumps(_safe_value(row), ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def _sort_rows(rows: Any) -> Any:
|
|
if isinstance(rows, list):
|
|
return sorted(rows, key=_row_sort_key)
|
|
return rows
|
|
|
|
|
|
def _is_optional_unsupported(name: str, error: Exception) -> bool:
|
|
if isinstance(error, PermissionError):
|
|
return False
|
|
if name not in {"packages", "procedure_parameters", "generators", "dependencies", "monitoring"}:
|
|
return False
|
|
if isinstance(error, AttributeError):
|
|
return True
|
|
message = str(error).lower()
|
|
if any(
|
|
marker in message
|
|
for marker in (
|
|
"permission",
|
|
"privilege",
|
|
"access denied",
|
|
"not authorized",
|
|
"unauthorized",
|
|
"authorization",
|
|
)
|
|
):
|
|
return False
|
|
if isinstance(error, NotImplementedError):
|
|
return True
|
|
return any(
|
|
marker in message
|
|
for marker in (
|
|
"table unknown",
|
|
"column unknown",
|
|
"does not exist",
|
|
"not found",
|
|
"unsupported",
|
|
"not supported",
|
|
"not implemented",
|
|
"capability unavailable",
|
|
)
|
|
)
|
|
|
|
|
|
def _fetch_rows(result: Any) -> list[Any]:
|
|
description = getattr(result, "description", None)
|
|
if hasattr(result, "fetchall"):
|
|
result = result.fetchall()
|
|
rows = list(result) if result is not None else []
|
|
if description:
|
|
names = [str(column[0]) for column in description]
|
|
return [
|
|
dict(zip(names, row)) if isinstance(row, tuple) else row
|
|
for row in rows
|
|
]
|
|
return rows
|
|
|
|
|
|
def _optional_operation(reader: ReadOnlyCatalogReader, method_name: str):
|
|
try:
|
|
operation = getattr(reader, method_name)
|
|
except AttributeError:
|
|
return None
|
|
if method_name.startswith("get_") and getattr(type(reader), method_name, None) is getattr(ReadOnlyCatalogReader, method_name, None):
|
|
try:
|
|
getattr(reader, "list_" + method_name.removeprefix("get_"))
|
|
except AttributeError:
|
|
return None
|
|
return operation
|
|
|
|
|
|
def collect_metadata(reader: ReadOnlyCatalogReader) -> dict:
|
|
"""Collect catalog rows through the approved structured reader only."""
|
|
from .catalog import StructuredFirebirdReader
|
|
|
|
if not isinstance(reader, ReadOnlyCatalogReader):
|
|
raise TypeError("reader must be ReadOnlyCatalogReader")
|
|
if any(callable(getattr(reader, name, None)) for name in ("execute", "query", "raw_execute", "run_sql")):
|
|
raise TypeError("reader must not expose raw execution")
|
|
is_approved_fixture = type(reader) in _APPROVED_FIXTURE_TYPES
|
|
structured_methods = (
|
|
"get_database_info", "get_relation_columns", "get_primary_keys",
|
|
"get_foreign_keys", "get_indices", "get_triggers", "get_procedures",
|
|
"get_procedure_parameters", "get_packages", "get_generators", "get_dependencies",
|
|
"get_monitoring",
|
|
)
|
|
approved_implementations = {
|
|
getattr(ReadOnlyCatalogReader, name)
|
|
for name in structured_methods
|
|
} | {
|
|
getattr(StructuredFirebirdReader, name)
|
|
for name in structured_methods
|
|
}
|
|
if not is_approved_fixture and (
|
|
type(reader) is not StructuredFirebirdReader
|
|
or any(getattr(type(reader), name, None) not in approved_implementations for name in structured_methods)
|
|
):
|
|
raise TypeError("reader must use the sealed structured operations")
|
|
metadata = {}
|
|
errors = []
|
|
unavailable = ["events"]
|
|
relations = []
|
|
|
|
def run(name, operation, *, optional=False):
|
|
try:
|
|
return operation()
|
|
except Exception as error:
|
|
kind = "optional_unsupported" if optional and _is_optional_unsupported(name, error) else "query_error"
|
|
errors.append({"query": name, "kind": kind, "error": type(error).__name__})
|
|
if kind == "optional_unsupported":
|
|
unavailable.append(name)
|
|
return []
|
|
|
|
metadata["database"] = run("database", reader.get_database_info)
|
|
relations = run("relations", reader.list_relations)
|
|
metadata["relations"] = relations
|
|
metadata["fields"] = []
|
|
|
|
def extend_unique(target, rows):
|
|
for row in rows:
|
|
if row not in target:
|
|
target.append(row)
|
|
|
|
for relation_row in relations:
|
|
if not isinstance(relation_row, Mapping):
|
|
continue
|
|
relation = relation_row.get("RELATION_NAME", relation_row.get("relation_name"))
|
|
if relation is None:
|
|
continue
|
|
extend_unique(metadata["fields"], run("fields", lambda relation=relation: reader.get_relation_columns(relation)))
|
|
metadata["keys"] = []
|
|
metadata["foreign_keys"] = []
|
|
metadata["indexes"] = []
|
|
for relation_row in relations:
|
|
if not isinstance(relation_row, Mapping):
|
|
continue
|
|
relation = relation_row.get("RELATION_NAME", relation_row.get("relation_name"))
|
|
if relation is None:
|
|
continue
|
|
extend_unique(metadata["keys"], run("keys", lambda relation=relation: reader.get_primary_keys(relation)))
|
|
extend_unique(metadata["foreign_keys"], run("foreign_keys", lambda relation=relation: reader.get_foreign_keys(relation)))
|
|
extend_unique(metadata["indexes"], run("indexes", lambda relation=relation: reader.get_indices(relation)))
|
|
|
|
operations = {
|
|
"triggers": (reader.get_triggers, False),
|
|
"procedures": (reader.get_procedures, False),
|
|
"procedure_parameters": (_optional_operation(reader, "get_procedure_parameters"), True),
|
|
"packages": (_optional_operation(reader, "get_packages"), True),
|
|
"generators": (_optional_operation(reader, "get_generators"), True),
|
|
"dependencies": (_optional_operation(reader, "get_dependencies"), True),
|
|
"monitoring": (_optional_operation(reader, "get_monitoring"), True),
|
|
}
|
|
for name, (operation, optional) in operations.items():
|
|
if operation is None or not callable(operation):
|
|
metadata[name] = []
|
|
unavailable.append(name)
|
|
continue
|
|
metadata[name] = run(name, operation, optional=optional)
|
|
|
|
for name, rows in list(metadata.items()):
|
|
metadata[name] = _safe_value(_sort_rows(rows))
|
|
metadata["events"] = []
|
|
metadata["unavailable"] = sorted(set(unavailable))
|
|
metadata["errors"] = sorted((_safe_value(error) for error in errors), key=_row_sort_key)
|
|
metadata["incomplete"] = bool(errors or metadata["unavailable"])
|
|
return metadata
|