254 lines
12 KiB
Python
254 lines
12 KiB
Python
from gti_discovery.mapping import build_schema_mapping, identify_product_candidates
|
|
|
|
|
|
def fixture_metadata():
|
|
return {
|
|
"relations": [
|
|
{"name": "PRODUCT", "approximate_count": 12},
|
|
{"name": "PRODUCT_STOCK", "approximate_count": 12},
|
|
{"name": "PRODUCT_IMAGE"},
|
|
],
|
|
"fields": [
|
|
{"relation": "PRODUCT", "name": "ID", "type": "INTEGER", "nullable": False},
|
|
{"relation": "PRODUCT", "name": "CODE", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "DESCRIPTION", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "NCM", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "IAT", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "IPPT", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "INCIDENCIA", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "CFOP_ENTRADA", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "CFOP_SAIDA", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "CST_ENTRADA", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "CST_SAIDA", "type": "VARCHAR"},
|
|
{"relation": "PRODUCT", "name": "IMAGE_DATA", "type": "BLOB", "subtype": 0},
|
|
{"relation": "PRODUCT_STOCK", "name": "PRODUCT_ID", "type": "INTEGER"},
|
|
{"relation": "PRODUCT_STOCK", "name": "QTY", "type": "DECIMAL"},
|
|
{"relation": "PRODUCT_IMAGE", "name": "PRODUCT_ID", "type": "INTEGER"},
|
|
{"relation": "PRODUCT_IMAGE", "name": "IMAGE_URL", "type": "VARCHAR"},
|
|
],
|
|
"keys": [
|
|
{"relation": "PRODUCT", "constraint_type": "PRIMARY KEY", "fields": ["ID"]},
|
|
],
|
|
"foreign_keys": [
|
|
{"child_relation": "PRODUCT_STOCK", "child_field": "PRODUCT_ID", "referenced_relation": "PRODUCT", "referenced_field": "ID"},
|
|
{"child_relation": "PRODUCT_IMAGE", "child_field": "PRODUCT_ID", "referenced_relation": "PRODUCT", "referenced_field": "ID"},
|
|
],
|
|
"indexes": [{"relation": "PRODUCT", "fields": ["CODE"], "unique": True}],
|
|
"triggers": [{"relation": "PRODUCT", "name": "BI_PRODUCT", "timing": "BEFORE", "event": "INSERT"}],
|
|
"generators": [{"name": "GEN_PRODUCT_ID"}],
|
|
"dependencies": [{"dependent": "BI_PRODUCT", "depended_on": "GEN_PRODUCT_ID"}],
|
|
}
|
|
|
|
|
|
def test_product_is_confirmed_only_when_structure_and_masked_scalar_samples_agree():
|
|
samples = {"PRODUCT": [{"ID": 1, "CODE": "P-001", "DESCRIPTION": "masked"}]}
|
|
candidates = identify_product_candidates(fixture_metadata(), samples)
|
|
|
|
product = next(item for item in candidates if item["relation"] == "PRODUCT")
|
|
assert product["classification"] == "CONFIRMED"
|
|
assert product["evidence"]["primary_key"] == ["ID"]
|
|
|
|
|
|
def test_mapping_prefers_primary_product_structure_over_auxiliary_product_table():
|
|
metadata = {
|
|
"relations": [
|
|
{"name": "PRODUTOS_AUX"},
|
|
{"name": "MATERIAL"},
|
|
],
|
|
"fields": [
|
|
{"relation": "PRODUTOS_AUX", "name": "ID", "type": "INTEGER"},
|
|
{"relation": "PRODUTOS_AUX", "name": "CODIGO", "type": "VARCHAR"},
|
|
{"relation": "MATERIAL", "name": "MATERIAL_CODIGO", "type": "VARCHAR"},
|
|
{"relation": "MATERIAL", "name": "MATERIAL_NOME", "type": "VARCHAR"},
|
|
{"relation": "MATERIAL", "name": "MATERIAL_UNIDADE", "type": "VARCHAR"},
|
|
{"relation": "MATERIAL", "name": "MATERIAL_ESTOQUE", "type": "DECIMAL"},
|
|
],
|
|
"keys": [
|
|
{"relation": "PRODUTOS_AUX", "constraint_type": "PRIMARY KEY", "fields": ["ID"]},
|
|
{"relation": "MATERIAL", "constraint_type": "PRIMARY KEY", "fields": ["MATERIAL_CODIGO"]},
|
|
],
|
|
}
|
|
result = build_schema_mapping(metadata, {
|
|
"PRODUTOS_AUX": [{"ID": 1, "CODIGO": "masked"}],
|
|
"MATERIAL": [{"MATERIAL_CODIGO": "masked", "MATERIAL_NOME": "masked", "MATERIAL_UNIDADE": "masked", "MATERIAL_ESTOQUE": 1}],
|
|
})
|
|
|
|
assert result["product"]["relation"] == "MATERIAL"
|
|
assert result["product"]["evidence"]["primary_key"] == ["MATERIAL_CODIGO"]
|
|
|
|
|
|
def test_inferred_product_preserves_structural_evidence():
|
|
metadata = {
|
|
"relations": [{"name": "MATERIAL"}],
|
|
"fields": [
|
|
{"relation": "MATERIAL", "name": "MATERIAL_CODIGO", "type": "VARCHAR"},
|
|
{"relation": "MATERIAL", "name": "MATERIAL_NOME", "type": "VARCHAR"},
|
|
],
|
|
"keys": [{"relation": "MATERIAL", "constraint_type": "PRIMARY KEY", "fields": ["MATERIAL_CODIGO"]}],
|
|
}
|
|
|
|
result = build_schema_mapping(metadata, {})
|
|
|
|
assert result["product"]["classification"] == "INFERRED"
|
|
assert result["product"]["evidence"]["primary_key"] == ["MATERIAL_CODIGO"]
|
|
|
|
|
|
def test_mapping_prefers_sampled_product_fields_over_empty_structural_table():
|
|
metadata = {
|
|
"relations": [{"name": "C000119"}, {"name": "CL00007"}],
|
|
"fields": [
|
|
{"relation": "C000119", "name": "CODIGO", "type": "INTEGER"},
|
|
{"relation": "C000119", "name": "NCM", "type": "VARCHAR"},
|
|
{"relation": "C000119", "name": "ESTOQUE", "type": "DECIMAL"},
|
|
{"relation": "CL00007", "name": "CODIGO", "type": "INTEGER"},
|
|
{"relation": "CL00007", "name": "PRODUTO", "type": "VARCHAR"},
|
|
{"relation": "CL00007", "name": "UNIDADE", "type": "VARCHAR"},
|
|
{"relation": "CL00007", "name": "MARCA", "type": "VARCHAR"},
|
|
{"relation": "CL00007", "name": "ESTOQUE", "type": "DECIMAL"},
|
|
],
|
|
"keys": [],
|
|
}
|
|
result = build_schema_mapping(metadata, {
|
|
"C000119": [],
|
|
"CL00007": [{"CODIGO": 276, "PRODUTO": "masked", "UNIDADE": "UN", "MARCA": "masked", "ESTOQUE": 1}],
|
|
})
|
|
|
|
assert result["product"]["relation"] == "CL00007"
|
|
|
|
|
|
def test_name_only_product_candidate_is_inferred_and_absence_is_unknown():
|
|
inferred = identify_product_candidates({"relations": [{"name": "PRODUCT"}], "fields": []}, {})
|
|
assert inferred[0]["classification"] == "INFERRED"
|
|
|
|
unknown = identify_product_candidates({"relations": [{"name": "LEDGER"}], "fields": []}, {})
|
|
assert unknown[0]["classification"] == "UNKNOWN"
|
|
|
|
|
|
def test_generic_primary_key_and_sample_do_not_confirm_unrelated_table():
|
|
metadata = {
|
|
"relations": [{"name": "PRODUCT"}],
|
|
"fields": [
|
|
{"relation": "PRODUCT", "name": "ID", "type": "INTEGER"},
|
|
{"relation": "PRODUCT", "name": "VALUE", "type": "VARCHAR"},
|
|
],
|
|
"keys": [{"relation": "PRODUCT", "constraint_type": "PRIMARY KEY", "fields": ["ID"]}],
|
|
}
|
|
|
|
candidate = identify_product_candidates(
|
|
metadata, {"PRODUCT": [{"ID": 1, "VALUE": "unrelated"}]}
|
|
)[0]
|
|
|
|
assert candidate["classification"] == "INFERRED"
|
|
assert candidate["evidence"]["semantic_fields"] == []
|
|
|
|
|
|
def test_mapping_covers_fiscal_media_stock_incremental_and_redacts_sensitive_samples():
|
|
samples = {"PRODUCT": [{"ID": 1, "NCM": "masked", "IMAGE_DATA": b"raw blob", "PASSWORD": "secret"}]}
|
|
result = build_schema_mapping(fixture_metadata(), samples)
|
|
|
|
assert result["product"]["classification"] == "CONFIRMED"
|
|
assert result["fiscal"]["fields"]["NCM"]["classification"] in {"CONFIRMED", "INFERRED"}
|
|
assert result["stock"]["relations"]["PRODUCT_STOCK"]["classification"] == "CONFIRMED"
|
|
assert result["media"]["fields"]["IMAGE_DATA"]["value"] == "<redacted>"
|
|
assert result["incremental"]["classification"] == "INFERRED"
|
|
assert result["triggers"][0]["source"] == "sanitized metadata only"
|
|
assert "raw blob" not in str(result)
|
|
assert "secret" not in str(result)
|
|
|
|
|
|
def test_mapping_recognizes_explicit_fiscal_concepts_and_operation_directions():
|
|
result = build_schema_mapping(
|
|
fixture_metadata(),
|
|
{"PRODUCT": [{
|
|
"ID": 1,
|
|
"IAT": "masked",
|
|
"IPPT": "masked",
|
|
"INCIDENCIA": "masked",
|
|
"CFOP_ENTRADA": "masked",
|
|
"CFOP_SAIDA": "masked",
|
|
"CST_ENTRADA": "masked",
|
|
"CST_SAIDA": "masked",
|
|
}]},
|
|
)
|
|
|
|
assert set(result["fiscal"]["fields"]) >= {
|
|
"IAT", "IPPT", "INCIDENCIA",
|
|
"CFOP_ENTRADA", "CFOP_SAIDA", "CST_ENTRADA", "CST_SAIDA",
|
|
}
|
|
|
|
|
|
def test_missing_mapping_evidence_is_unknown():
|
|
result = build_schema_mapping({"relations": [], "fields": []}, {})
|
|
|
|
assert result["product"]["classification"] == "UNKNOWN"
|
|
assert result["fiscal"]["classification"] == "UNKNOWN"
|
|
assert result["media"]["classification"] == "UNKNOWN"
|
|
assert result["incremental"]["classification"] == "UNKNOWN"
|
|
|
|
|
|
def collector_shaped_metadata():
|
|
return {
|
|
"relations": [
|
|
{"RDB$RELATION_NAME": "PRODUCT"},
|
|
{"RDB$RELATION_NAME": "PRODUCT_STOCK"},
|
|
{"RDB$RELATION_NAME": "PRODUCT_IMAGE"},
|
|
{"RDB$RELATION_NAME": "PRODUCT_FISCAL"},
|
|
],
|
|
"fields": [
|
|
{"RELATION_NAME": "PRODUCT", "FIELD_NAME": "ID", "TYPE_NAME": "INTEGER"},
|
|
{"RELATION_NAME": "PRODUCT", "FIELD_NAME": "DESCRIPTION", "TYPE_NAME": "VARCHAR"},
|
|
{"RELATION_NAME": "PRODUCT_STOCK", "FIELD_NAME": "PRODUCT_ID", "TYPE_NAME": "INTEGER"},
|
|
{"RELATION_NAME": "PRODUCT_STOCK", "FIELD_NAME": "QUANTITY", "TYPE_NAME": "DECIMAL"},
|
|
{"RELATION_NAME": "PRODUCT_IMAGE", "FIELD_NAME": "PRODUCT_ID", "TYPE_NAME": "INTEGER"},
|
|
{"RELATION_NAME": "PRODUCT_IMAGE", "FIELD_NAME": "IMAGE_DATA", "TYPE_NAME": "BLOB", "FIELD_SUB_TYPE": 0},
|
|
{"RELATION_NAME": "PRODUCT_IMAGE", "FIELD_NAME": "IMAGE_URL", "TYPE_NAME": "VARCHAR"},
|
|
{"RELATION_NAME": "PRODUCT_FISCAL", "FIELD_NAME": "PRODUCT_ID", "TYPE_NAME": "INTEGER"},
|
|
{"RELATION_NAME": "PRODUCT_FISCAL", "FIELD_NAME": "NCM", "TYPE_NAME": "VARCHAR"},
|
|
],
|
|
"keys": [{"RELATION_NAME": "PRODUCT", "CONSTRAINT_TYPE": "PRIMARY KEY", "FIELD_NAME": "ID"}],
|
|
"foreign_keys": [
|
|
{"CHILD_RELATION_NAME": "PRODUCT_STOCK", "CHILD_FIELD_NAME": "PRODUCT_ID", "REFERENCED_RELATION_NAME": "PRODUCT", "REFERENCED_FIELD_NAME": "ID"},
|
|
{"CHILD_RELATION_NAME": "PRODUCT_IMAGE", "CHILD_FIELD_NAME": "PRODUCT_ID", "REFERENCED_RELATION_NAME": "PRODUCT", "REFERENCED_FIELD_NAME": "ID"},
|
|
{"CHILD_RELATION_NAME": "PRODUCT_FISCAL", "CHILD_FIELD_NAME": "PRODUCT_ID", "REFERENCED_RELATION_NAME": "PRODUCT", "REFERENCED_FIELD_NAME": "ID"},
|
|
],
|
|
"triggers": [{"TRIGGER_NAME": "BI_PRODUCT", "RELATION_NAME": "PRODUCT", "TRIGGER_TYPE": 1}],
|
|
"dependencies": [{"DEPENDENT_NAME": "BI_PRODUCT", "DEPENDED_ON_NAME": "GEN_PRODUCT_ID"}],
|
|
"generators": [{"GENERATOR_NAME": "GEN_PRODUCT_ID"}],
|
|
}
|
|
|
|
|
|
def test_collector_shapes_map_separate_relations_and_correlate_dependencies():
|
|
result = build_schema_mapping(
|
|
collector_shaped_metadata(),
|
|
{"PRODUCT": [{"ID": 1, "DESCRIPTION": "customer item"}], "PRODUCT_IMAGE": [{"IMAGE_DATA": b"private"}]},
|
|
)
|
|
|
|
assert "PRODUCT_STOCK" in result["stock"]["relations"]
|
|
assert "PRODUCT_IMAGE" in result["media"]["relations"]
|
|
assert result["media"]["relations"]["PRODUCT_IMAGE"]["fields"]["IMAGE_DATA"]["subtype"] == 0
|
|
assert "PRODUCT_FISCAL" in result["fiscal"]["relations"]
|
|
assert result["triggers"][0]["trigger_type"] == 1
|
|
assert result["triggers"][0]["dependencies"] == ["GEN_PRODUCT_ID"]
|
|
|
|
|
|
def test_null_trigger_type_is_reported_as_unknown():
|
|
result = build_schema_mapping(
|
|
{"relations": [{"name": "PRODUCT"}], "fields": [], "triggers": [{"relation": "PRODUCT", "name": "BI_PRODUCT", "trigger_type": None}]},
|
|
{},
|
|
)
|
|
|
|
assert result["triggers"][0]["trigger_type"] == "UNKNOWN"
|
|
|
|
|
|
def test_samples_expose_only_allowlisted_structural_evidence():
|
|
result = build_schema_mapping(
|
|
fixture_metadata(),
|
|
{"PRODUCT": [{"ID": 1, "CODE": "customer-commercial-code", "DESCRIPTION": "customer name", "NCM": "12345678"}]},
|
|
)
|
|
|
|
serialized = str(result)
|
|
assert "customer-commercial-code" not in serialized
|
|
assert "customer name" not in serialized
|
|
assert "12345678" not in serialized
|
|
assert result["product"]["evidence"]["sample_fields"] == ["CODE", "DESCRIPTION", "ID", "NCM"]
|