match.py

Abgleich altes edwiki.de ↔ Frontier-IDs (Ergebnis match.json, 5,7 MB – nicht im Wiki, lässt sich aus Backup + galnet_raw neu erzeugen).

Ablage beim Arbeiten: tools/match.py. Herunterladen: auf den Dateinamen über dem Code klicken.

match.py

import json,csv,re,sys,collections,difflib
from pathlib import Path
from datetime import date,timedelta
R=Path(sys.argv[1]); old=json.load(open(sys.argv[2]))
def load(lang):
    d={}
    for f in (R/lang).glob('*.json'):
        for a in json.load(open(f,encoding='utf-8'))['articles']: d[a['id']]=a
    return d
EN,DE=load('EN'),load('DE'); ANY={}
for L in ['RU','PT-BR','FR','ES','EN-US','DE','EN']: ANY.update(load(L))
idx=list(csv.DictReader(open(R/'index.csv',encoding='utf-8'),delimiter=';'))
byday=collections.defaultdict(list)
for r in idx: byday[r['date']].append(r)
norm=lambda s: re.sub(r'\W+',' ',re.sub(r'<[^>]+>|\*\*|//','',s or '')).lower().strip()
def has_de(i): return i in DE and i in EN and DE[i]['body']!=EN[i]['body'] or (i in DE and i not in EN)
def keys(a):
    t=a['title']+' '+a['body']
    k=set(re.findall(r'\b[A-Z][a-zA-Z\'-]{3,}',a['body']))|set(re.findall(r'\b\d[\d,.]*\d\b|\b\d\b',t))
    k={re.sub(r'[,.]','',x).lower() for x in k}
    stop={'the','this','that','federal','federation','empire','imperial','alliance','galnet','pilots','commander','commanders','however','meanwhile','but','and','it','in','a','we','i','he','she','they'}
    return {x for x in k if x not in stop}
def score(o,r):
    i=r['frontier_id']; a=ANY[i]
    s=0
    if has_de(i):
        de=DE[i]
        s=max(difflib.SequenceMatcher(None,norm(o['title']),norm(de['title'])).ratio(),
              difflib.SequenceMatcher(None,norm(o['body'])[:400],norm(de['body'])[:400]).ratio())
    k=keys(a); ot=norm(o['title']+' '+o['body']).replace(' ','')
    if k:
        hit=sum(1 for x in k if re.sub(r'\W','',x) in ot)
        s=max(s,hit/len(k))
    return s
pairs=[]
for oi,o in enumerate(old):
    d=date.fromisoformat(o['date'][:10])
    for k in range(-3,4):
        for r in byday.get(str(d+timedelta(days=k)),[]):
            pairs.append((score(o,r)-abs(k)*0.04,oi,r['frontier_id']))
pairs.sort(reverse=True); uo=set(); uf=set(); res={}
for sc,oi,fid in pairs:
    if sc<0.3: break
    if oi in uo or fid in uf: continue
    uo.add(oi); uf.add(fid); res[oi]=(fid,round(sc,2))
out=[]
for oi,o in enumerate(old):
    fid,sc=res.get(oi,(None,0)); out.append(dict(o,fid=fid,score=sc,de_frontier=bool(fid and has_de(fid))))
json.dump(out,open(sys.argv[3],'w'),ensure_ascii=False,indent=1)
m=[x for x in out if x['fid']]
print('alt',len(out),'zugeordnet',len(m),'ohne',len(out)-len(m), collections.Counter(x['date'][:4] for x in out if not x['fid']))
miss=[r for r in idx if r['frontier_id'] not in uf]
print('Frontier ohne alte Seite',len(miss),collections.Counter(r['date'][:4] for r in miss))
noDE=[r for r in idx if not has_de(r['frontier_id'])]
print('ohne Frontier-DE',len(noDE),'davon alte Übersetzung',sum(1 for r in noDE if r['frontier_id'] in uf), collections.Counter(r['date'][:4] for r in noDE if r['frontier_id'] not in uf))
print('Scores',sorted(collections.Counter(round(x['score'],1) for x in m).items()))