47 lines
1.6 KiB
Python
47 lines
1.6 KiB
Python
"""Runs INSIDE the fiscal-hub container (read-only on PostgreSQL): classifies [ncm, description] pairs with the
|
|
production Classifier through a memoizing repository wrapper. Same code path as /v1/classify, minus HTTP and the
|
|
classification_cache write; the wrapper only memoizes pure dataset lookups (datasets are static during the run).
|
|
usage: python /tmp/bulk_classify_in_container.py /tmp/in.json /tmp/out.json
|
|
"""
|
|
import json
|
|
import sys
|
|
from functools import lru_cache
|
|
|
|
from central.app.classification import Classifier
|
|
from central.app.datasets.service import DatasetService
|
|
from central.app.db import SessionLocal
|
|
|
|
|
|
class Memo:
|
|
def __init__(self, service):
|
|
self._s = service; self._cache = {}
|
|
|
|
def __getattr__(self, name):
|
|
attr = getattr(self._s, name)
|
|
if not callable(attr):
|
|
return attr
|
|
def wrapped(*a, **kw):
|
|
key = (name, a, tuple(sorted(kw.items())))
|
|
try:
|
|
hash(key)
|
|
except TypeError:
|
|
return attr(*a, **kw)
|
|
if key not in self._cache:
|
|
self._cache[key] = attr(*a, **kw)
|
|
return self._cache[key]
|
|
return wrapped
|
|
|
|
|
|
def main():
|
|
pairs = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
out = []
|
|
with SessionLocal() as session:
|
|
classifier = Classifier(Memo(DatasetService(session)))
|
|
for ncm, desc in pairs:
|
|
out.append(classifier.classify({"current_ncm": ncm, "description": desc}))
|
|
json.dump(out, open(sys.argv[2], "w", encoding="utf-8"), ensure_ascii=False, default=str)
|
|
print(len(out))
|
|
|
|
|
|
main()
|