#!/usr/bin/env python3 """Newsletter-Paket für edwiki-import.php bauen. Eingaben (Arbeitsordner): en/en_NNN.txt bereinigter englischer Rumpf (clean_en.py) de/de_NNN.txt deutsche Übersetzung (Claude) raw/images_NNN.json Bildliste (html2wiki.py) newsletter_index.json Nummer, Datum, Original-Link (galnet.fr-Liste) meta.json je Nummer: author, wayback, related, translation Ausgabe: build/pages/..., build/media/..., build/newsletter_NNN-NNN.csv, pakete/newsletter_paket_-_tN.zip (je <= 3,7 MB, CSV im letzten Teil) Aufruf: build_newsletter.py """ import csv import io import json import os import re import shutil import sys import zipfile from PIL import Image MAXPX = 1920 ZIPMAX = 3_700_000 KI_NOTE = '//Übersetzung aus dem Englischen: Claude, ungeprüft.//' LANGS = {'de': 'news:newsletter', 'en': 'en:news:newsletter'} def frame(title, body, footer): return f"""[<>] ====== {title} ====== ---- {body.strip()} {footer} {{{{structbox>newsletter}}}} ---- [<>] """ def de_date(iso): y, m, d = iso.split('-') return f'{d}.{m}.{y}' def links(text, lang): ns = LANGS[lang] text = text.replace('[[NL:start|', '[[news:newsletter:start|') # Übersicht gibt es nur auf Deutsch return re.sub(r'\[\[NL:(\d{3})\|', lambda m: f'[[{ns}:{m.group(1)}|', text) def save_image(src, dst): try: Image.open(src).verify() except Exception: return 'FEHLT (Wayback-Kopie defekt)' os.makedirs(os.path.dirname(dst), exist_ok=True) ext = os.path.splitext(dst)[1] if ext == '.jpg': im = Image.open(src) if max(im.size) > MAXPX: im.thumbnail((MAXPX, MAXPX), Image.LANCZOS) im.convert('RGB').save(dst, 'JPEG', quality=88, optimize=True) return 'verkleinert' shutil.copyfile(src, dst) return '' def main(): work, a, b = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) idx = {r['nr']: r for r in json.load(open(os.path.join(work, 'newsletter_index.json')))} meta = json.load(open(os.path.join(work, 'meta.json'))) build = os.path.join(work, 'build') shutil.rmtree(build, ignore_errors=True) rows, report, gaps = [], [], [] for n in range(a, b + 1): nr = '%03d' % n info, m = idx[str(n)], meta[nr] date = info['date'] title = f'Elite: Dangerous Newsletter #{n}' src = f'//Quelle: Frontier Developments, {title}, {de_date(date)}//' de = links(open(os.path.join(work, 'de', f'de_{nr}.txt')).read(), 'de') en = links(open(os.path.join(work, 'en', f'en_{nr}.txt')).read(), 'en') pages = { f'pages/news/newsletter/{nr}.txt': frame(title, de, KI_NOTE + ' \\\\\n' + src), f'pages/en/news/newsletter/{nr}.txt': frame(title, en, f'//Source: Frontier Developments, {title}, {date}//'), } for p, t in pages.items(): os.makedirs(os.path.dirname(os.path.join(build, p)), exist_ok=True) open(os.path.join(build, p), 'w').write(t) # Bilder: alle, die im Text vorkommen + Kopfbild (Box) used = set(re.findall(r'\{\{news:newsletter:%s:([^?|}]+)' % nr, de)) imgs = json.load(open(os.path.join(work, 'raw', f'images_{nr}.json'))) head = imgs[0]['name'] if m.get('header_local'): # Wayback hat ein falsches Kopfbild geliefert (gleicher Dateiname bei Mailchimp) imgs[0]['local'] = m['header_local'] report.append(f'#{n} Kopfbild ersetzt durch {os.path.basename(m["header_local"])}') for im in imgs: if im['name'] in used or im['name'] == head: note = save_image(im['local'], os.path.join(build, 'media/news/newsletter', nr, im['name'])) if note.startswith('FEHLT'): gaps.append(f'#{n}\tnews:newsletter:{nr}:{im["name"]}\t{im["orig"] or "-"}') elif note: report.append(f'#{n} {im["name"]}: {note} auf {MAXPX} px') missing = used - {i['name'] for i in imgs} if missing: report.append(f'#{n} FEHLENDE BILDER: {sorted(missing)}') rows.append({ 'pid': f'news:newsletter:{nr}', 'number': str(n), 'date': date, 'picture': f'news:newsletter:{nr}:{head}', 'author': m['author'], 'url': re.sub(r'&e=[^&]*', '', info['url']), # Abonnenten-Kennung entfernen 'archive': m['wayback'], 'translation': m.get('translation', 'KI'), 'relations.related': ','.join(m.get('related', [])), }) # feste Seiten (Übersicht, Vorlage, news:start, Sidebar): static/pages/... 1:1 übernehmen shutil.copytree(os.path.join(work, 'static', 'pages'), os.path.join(build, 'pages'), dirs_exist_ok=True) if gaps: open(os.path.join(build, f'luecken_newsletter_{a:03d}-{b:03d}.txt'), 'w').write( 'Nr\tfehlendes Bild (Medien-ID)\tOriginal (über Wayback suchen)\n' + '\n'.join(gaps) + '\n') report.append(f'{len(gaps)} Bilder fehlen, siehe luecken_newsletter_{a:03d}-{b:03d}.txt') csvname = f'newsletter_{a:03d}-{b:03d}.csv' with open(os.path.join(build, csvname), 'w', newline='') as fh: w = csv.DictWriter(fh, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) # Pakete files = [] for root, _, fs in os.walk(build): for f in sorted(fs): full = os.path.join(root, f) rel = os.path.relpath(full, build) if rel.endswith('.csv'): continue files.append((rel, full)) files.sort() outdir = os.path.join(work, 'pakete') shutil.rmtree(outdir, ignore_errors=True) os.makedirs(outdir) parts, cur, size = [], [], 0 for rel, full in files: s = os.path.getsize(full) + 200 + len(rel) * 2 if cur and size + s > ZIPMAX: parts.append(cur) cur, size = [], 0 cur.append((rel, full)) size += s cur.append((csvname, os.path.join(build, csvname))) parts.append(cur) for i, part in enumerate(parts, 1): zn = os.path.join(outdir, f'newsletter_paket_{a:03d}-{b:03d}_t{i}.zip') with zipfile.ZipFile(zn, 'w', zipfile.ZIP_DEFLATED) as z: for rel, full in part: z.write(full, rel) report.append(f'{os.path.basename(zn)}: {len(part)} Dateien, {os.path.getsize(zn) / 1e6:.2f} MB') print('\n'.join(report)) if __name__ == '__main__': main()