Erzeugt aus galnet_raw/ + Schlagwörtern das Import-Paket (DE-Seiten, Sprachseiten, Übersichten, Bilder, CSV, Lückenbericht).
Ablage beim Arbeiten: build_galnet.py. Herunterladen: auf den Dateinamen über dem Code klicken.
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ build_galnet.py - erzeugt aus galnet_raw/ das Import-Paket fuer edwiki (news:galnet). python3 build_galnet.py --raw galnet_raw --year 2025 2026 --out paket Eingaben galnet_raw/ Ausgabe von galnet_fetch.py (JSON je Sprache/Monat, index.csv, images/) tags/<JJJJ>.json Schlagwoerter je Frontier-ID (von Claude gepflegt) translations/<JJJJ>.json eigene/KI-Uebersetzungen je Frontier-ID {"title","body","source"} old_match.json (optional) Zuordnung altes edwiki -> Frontier-ID (Bilder fuer Meldungen ohne Frontier-Bild) Ausgabe (--out) pages/news/galnet/... DE-Seiten, Monats-/Jahresseiten, overview, start pages/<sprache>/news/galnet/... Sprachseiten (en, en-us, es, fr, pt-br, ru) media/news/galnet/*.jpg benoetigte Bilder (max. 1920 px) galnet_<JJJJ>.csv struct-CSV-Import (Schema galnet, Art "Seite") related_<JJJJ>.json Nachtraege fuer relations.related (nach dem CSV-Import) luecken_<JJJJ>.txt Lueckenbericht (Wiki-Syntax) """ import argparse, csv, json, re, sys from collections import defaultdict from pathlib import Path NS = 'news:galnet' YEAR_OFFSET = 1286 LANGS = [('EN', 'en'), ('EN-US', 'en-us'), ('ES', 'es'), ('FR', 'fr'), ('PT-BR', 'pt-br'), ('RU', 'ru')] MON = ['', 'Januar', 'Februar', 'März', 'April', 'Mai', 'Juni', 'Juli', 'August', 'September', 'Oktober', 'November', 'Dezember'] GENERIC = 'news:galnet:galnet_generic.jpg' TAG_FIELDS = ['factions', 'corporations', 'persons', 'systems', 'codex', 'storyline'] KI_NOTE = '//Übersetzung aus dem Englischen: Claude, ungeprüft.//' # ------------------------------------------------------------------ Daten def load_lang(raw, lang, years): d = {} for f in sorted((raw / lang).glob('*.json')): if int(f.stem[:4]) - YEAR_OFFSET not in years: continue for a in json.loads(f.read_text(encoding='utf-8'))['articles']: d[a['id']] = a return d def media_id(path_or_name): stem = Path(path_or_name).stem.lower() return f'{NS}:' + re.sub(r'[^a-z0-9_]', '_', stem) + '.jpg' def body2wiki(b): b = (b or '').replace('\r\n', '\n').replace('<Titan8>', 'Cocijo') lines = [] for line in (l.strip() for l in b.split('\n')): if not line: continue line = re.sub(r'\*([^*\n]+)\*', r'//\1//', line) if line.startswith('•') or re.match(r'^[-–] ', line): lines.append(' * ' + line.lstrip('•-– ').strip()) else: lines.append(line) out = [] for i, l in enumerate(lines): if i and not (l.startswith(' * ') and lines[i - 1].startswith(' * ')): out.append('') out.append(l) return '\n'.join(out) def clean_title(t): return re.sub(r'\s+', ' ', t or '').strip() # ------------------------------------------------------------------ Seiten TABS = f"""<WRAP tabs> * [[{NS}:start|Aktuelle Galnet News]] * [[{NS}:overview|Übersicht (chronologisch)]] * [[{NS}:archiv|Archiv mit Suche]] </WRAP>""" def frame(title, body): return f"""<WRAP centeralign> ====== {title} ====== </WRAP> [<>] ---- {TABS} {body} ---- [<>] """ def month_tabs(y): return '<WRAP tabs>\n' + '\n'.join( f' * [[{NS}:{y}:{k:02d}:start|{MON[k][:3]}]]' for k in range(1, 13)) + '\n</WRAP>' def struct_table(flt): return f"""---- struct table ---- schema: galnet cols: date_ingame, picture, %title% head: Datum, Bild, Meldung filter: %pageid% ~ {flt} sort: date, %pageid% ----""" def article_page(title, body, date_ingame, note=''): note = f'\n\n{note}' if note else '' return f"""[<>] <WRAP centeralign> ====== {title} ====== </WRAP> ---- <WRAP group> <WRAP twothirds column box round> <blockquote> {body} <cite>GalNet, {date_ingame}</cite> </blockquote>{note} </WRAP> <WRAP third column box round> {{{{structbox>galnet}}}} </WRAP> </WRAP> ---- [<>] """ def year_page(y): iy = y + YEAR_OFFSET return frame(str(iy), f"""<WRAP center round box 90%> <WRAP centeralign>[[{NS}:overview|Übersicht]] » {iy} //(Echtzeit {y})//</WRAP> {month_tabs(y)} {struct_table(f'{NS}:{y}:*')} </WRAP>""") def month_page(y, m): iy = y + YEAR_OFFSET return frame(f'{MON[m]} {iy}', f"""<WRAP center round box 90%> <WRAP centeralign>[[{NS}:overview|Übersicht]] » [[{NS}:{y}:start|{iy}]] » {MON[m]}</WRAP> {month_tabs(y)} {struct_table(f'{NS}:{y}:{m:02d}:*')} </WRAP>""") def overview_page(years): rows = '\n'.join( f'| [[{NS}:{y}:start|{y + YEAR_OFFSET}]] | ' + ' | '.join(f'[[{NS}:{y}:{k:02d}:start|{MON[k][:3]}]]' for k in range(1, 13)) + ' |' for y in sorted(years, reverse=True)) return frame('Übersicht (chronologisch)', f"""<WRAP center round box 90%> ^ Jahr ^ Monate ^^^^^^^^^^^^ {rows} </WRAP>""") def start_page(pid, title, body, date_ingame, picture): pic = f'{{{{{picture}?450|}}}}' if picture else '' return frame('GalNet', f"""<WRAP group> <WRAP half column round box> ===== Aktuelle Galnet News ===== **[[{pid}|{title}]]** \\\\ //{date_ingame}// {body} </WRAP> <WRAP half column box round> {pic} </WRAP> </WRAP>""") # ------------------------------------------------------------------ Hauptteil def main(): ap = argparse.ArgumentParser() ap.add_argument('--raw', required=True) ap.add_argument('--year', type=int, nargs='+', required=True, help='Echtzeit-Jahre, z. B. 2025 2026') ap.add_argument('--out', required=True) ap.add_argument('--tags', default='tags') ap.add_argument('--translations', default='translations') ap.add_argument('--old-match', default=None) ap.add_argument('--old-media', default=None, help='Ordner mit Olivers alten Galnet-Bildern') ap.add_argument('--all-years', type=int, nargs='*', default=[], help='alle Jahre im Wiki (fuer overview)') ap.add_argument('--start', action='store_true', help='Startseite mit neuester Meldung schreiben') a = ap.parse_args() raw, out = Path(a.raw), Path(a.out) years = set(a.year) idx = [r for r in csv.DictReader(open(raw / 'index.csv', encoding='utf-8'), delimiter=';') if int(r['date'][:4]) in years] data = {L: load_lang(raw, L, years) for L in ['EN', 'DE'] + [l for l, _ in LANGS if l != 'EN']} EN, DE = data['EN'], data['DE'] tags, trans = {}, {} for y in years: f = Path(a.tags) / f'{y}.json' if f.exists(): tags.update(json.loads(f.read_text(encoding='utf-8'))) f = Path(a.translations) / f'{y}.json' if f.exists(): trans.update(json.loads(f.read_text(encoding='utf-8'))) old = {} if a.old_match: for o in json.loads(Path(a.old_match).read_text(encoding='utf-8')): if o.get('fid'): old[o['fid']] = o rows, related, needed_media = [], {}, {} raw_stems = {f.stem.lower() for f in (raw / 'images').iterdir()} if (raw / 'images').exists() else set() report = defaultdict(list) lang_count = defaultdict(int) newest = None for r in idx: fid = r['frontier_id'] pid = f"{NS}:{r['date'][:4]}:{r['date'][5:7]}:{r['date'][8:10]}_{r['number']}" y, m, d = r['date_ingame'].split('-') date_ingame = f'{d}.{m}.{y}' en = EN.get(fid) src = en or next(data[L][fid] for L in data if fid in data[L]) # --- Deutsch de = DE.get(fid) note = '' if de and not (en and de['body'] == en['body']): title, body, tstate = de['title'], de['body'], 'Frontier' elif fid in trans: t = trans[fid] title, body = t['title'], t['body'] tstate = t.get('source', 'KI') if tstate == 'KI': note = KI_NOTE else: title, body, tstate = src['title'], src['body'], 'fehlt' report['DE fehlt (Originaltext eingesetzt)'].append(pid) if de and en and de['body'] == en['body']: report['DE von Frontier unübersetzt'].append(pid) if '<Titan' in (body or ''): report['Platzhalter im Frontier-Text korrigiert'].append(pid) title = clean_title(title) write(out / 'pages' / (pid.replace(':', '/') + '.txt'), article_page(title, body2wiki(body), date_ingame, note)) # --- Sprachseiten for L, code in LANGS: x = data[L].get(fid) if not x: if L in ('EN', 'EN-US') and en is None: pass elif r['date'] >= '2016-01-01' and L not in ('EN', 'EN-US') and fid in DE: report[f'{L} fehlt (DE vorhanden)'].append(pid) continue if L not in ('EN', 'EN-US') and en and x['body'] == en['body']: report[f'{L} von Frontier unübersetzt'].append(pid) continue write(out / 'pages' / code / (pid.replace(':', '/') + '.txt'), article_page(clean_title(x['title']), body2wiki(x['body']), date_ingame)) lang_count[code] += 1 # --- Bild picture = '' fr_ok = bool(src.get('images')) and Path(src['images'][0]['path']).stem.lower() in raw_stems if fr_ok: p = src['images'][0]['path'] picture = media_id(p) needed_media[picture] = ('frontier', Path(p).stem) elif src.get('images') and not (fid in old and old[fid].get('images')): picture = GENERIC needed_media[picture] = ('old', 'galnet_generic.jpg') report['Frontier-Bild nicht abrufbar, Platzhalter galnet_generic'].append(pid) elif fid in old and old[fid].get('images'): picture = media_id(old[fid]['images'][0].split(':')[-1]) needed_media[picture] = ('old', old[fid]['images'][0].split(':')[-1]) report['Bild aus dem alten edwiki'].append(pid) else: report['ohne Bild'].append(pid) # --- struct tg = tags.get(fid) if tg is None: report['nicht verschlagwortet'].append(pid) tg = {} rows.append(dict(pid=pid, date=r['date'], date_ingame=date_ingame, picture=picture, frontier_id=fid, url='https://www.elitedangerous.com/news/galnet/' + ((de or src).get('slug') or ''), translation=tstate, **{k: ', '.join(tg.get(k, [])) for k in TAG_FIELDS})) if tg.get('related'): related[pid] = tg['related'] if newest is None or (r['date'], pid) > (newest[0], newest[1]): newest = (r['date'], pid, title, body2wiki(body), date_ingame, picture) # --- Monats-/Jahresseiten, Uebersicht, Start for y in years: write(out / 'pages' / NS.replace(':', '/') / str(y) / 'start.txt', year_page(y)) for m in range(1, 13): write(out / 'pages' / NS.replace(':', '/') / str(y) / f'{m:02d}' / 'start.txt', month_page(y, m)) write(out / 'pages' / NS.replace(':', '/') / 'overview.txt', overview_page(set(a.all_years) | years)) if a.start and newest: write(out / 'pages' / NS.replace(':', '/') / 'start.txt', start_page(*newest[1:])) # --- Bilder raw_img = raw / 'images' from PIL import Image for mid, (kind, name) in sorted(needed_media.items()): srcf = None if kind == 'frontier': srcf = next((f for f in raw_img.iterdir() if f.stem.lower() == name.lower()), None) elif a.old_media: srcf = next(iter(Path(a.old_media).glob(name)), None) if not srcf: report['Bilddatei fehlt'].append(mid) continue target = out / 'media' / (mid.replace(':', '/')) target.parent.mkdir(parents=True, exist_ok=True) im = Image.open(srcf).convert('RGB') if im.width > 1920: im = im.resize((1920, round(im.height * 1920 / im.width)), Image.LANCZOS) im.save(target, quality=85) # --- CSV, related, Bericht tag = '_'.join(str(y) for y in sorted(years)) with open(out / f'galnet_{tag}.csv', 'w', encoding='utf-8', newline='') as f: w = csv.DictWriter(f, fieldnames=['pid', 'date', 'date_ingame', 'picture', 'frontier_id', 'url', 'translation'] + TAG_FIELDS) w.writeheader() w.writerows(rows) write(out / f'related_{tag}.json', json.dumps(related, ensure_ascii=False, indent=1)) lines = [f'===== Lückenbericht {", ".join(str(y + YEAR_OFFSET) for y in sorted(years))} =====', '', f'Meldungen: {len(rows)} · Sprachseiten: ' + ', '.join(f'{k} {v}' for k, v in sorted(lang_count.items())), ''] for k, v in report.items(): lines.append(f'==== {k} ({len(v)}) ====') lines += [f' * [[{x}]]' if x.startswith(NS) and not x.endswith('.jpg') else " * ''" + x + "''" for x in v] lines.append('') write(out / f'luecken_{tag}.txt', '\n'.join(lines)) print(f'Meldungen {len(rows)}, Sprachseiten {dict(lang_count)}, Bilder {len(needed_media)}') for k, v in report.items(): print(f' {k}: {len(v)}') def write(path, text): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding='utf-8') if __name__ == '__main__': main()