#!/usr/bin/env python3
"""Rohtext (html2wiki) -> bereinigter englischer Seitenrumpf.

- Inhaltsverzeichnis: Überschrift -> fette Zeile, ohne "Follow Elite: Dangerous", Banner bleibt
- Links auf andere Newsletter (Mailchimp-IDs) -> interne Links
- E-Mail-Adressen nicht fett, kleine Leerzeichen-/Satzzeichenfehler
- gezielte Einzelkorrekturen (FIXES)
Aufruf: clean_en.py <raw-ordner> <index.json> <ausgabe-ordner>
"""
import json
import os
import re
import sys

FIXES = {
    '015': [('**[[http://elite.frontier.co.uk/|http://elite.frontier.co.u]]k**', '**[[http://elite.frontier.co.uk/|http://elite.frontier.co.uk]]**'),
            ('**Engines**//: "When', '**Engines:** //"When')],
    '002': [('but**[[', 'but **[['), (']]**to check', ']]** to check')],
    '003': [('**[[http://i1305.photobucket.com/albums/s553/FrontierAshley/EmpireshipsketchesCropped_zpsc19947fb.jpg|smaller Imperial Ships]]**',
             '**[[NL:002|smaller Imperial Ships]]**')],
    '004': [('**[[http://s1305.photobucket.com/user/FrontierAshley/media/FedandEmpirefighters_zps35212663.jpg.html|early concept thumbnails]]**',
             '**[[NL:001|early concept thumbnails]]**'),
            ('\nShield Systems:\n', '\n**Shield Systems:**\n')],
    '007': [('&e=[UNIQID]', '')],
    '008': [(' \\\\ --- \\\\ This week’s drabble', '\n\n===== Elite Fiction: Childish Dreams =====\n\nThis week’s drabble'),
            ('**[[http://www.laveradio.com/|their site]]**now', '**[[http://www.laveradio.com/|their site]]** now')],
    '009': [('**“**"We don\'t need the whole universe, just the Milky Way." Where to start?',
             '“We don\'t need the whole universe, just the Milky Way.” Where to start?'),
            ('ring system(s). **”**', 'ring system(s).'),
            ('**[[http://elite.frontier.co.uk/images/newsletter_2013_07_05/schematic_flat_empfigher.png|Empire]][[http://elite.frontier.co.uk/images/newsletter_2013_07_05/schematic_flat_empfigher.png|Fighter]]**',
             '**[[http://elite.frontier.co.uk/images/newsletter_2013_07_05/schematic_flat_empfigher.png|Empire Fighter]]**'),
            ('\n- Elsewhere in the Elite galaxy', '\nElsewhere in the Elite galaxy')],
}


def main():
    rawdir, index, outdir = sys.argv[1:4]
    idx = json.load(open(index))
    by_id = {}
    for r in idx:
        m = re.search(r'id=([0-9a-f]{10})', r['url'])
        if m and re.fullmatch(r'\d+', r['nr']):
            by_id.setdefault(m.group(1), '%03d' % int(r['nr']))
    os.makedirs(outdir, exist_ok=True)
    for fn in sorted(os.listdir(rawdir)):
        m = re.match(r'en_(\d{3})\.txt$', fn)
        if not m:
            continue
        nr = m.group(1)
        t = open(os.path.join(rawdir, fn)).read()
        # Inhaltsverzeichnis
        t = re.sub(r'===== Table of Contents[^=]*=====', '**Contents:**', t)
        t = re.sub(r'\n  \* (\*\*)?Follow Elite: Dangerous(\*\*)?\n', '\n', t)
        # persönliche Anrede aus dem Mailchimp-Seriendruck (Name des Abonnenten) neutralisieren
        t = re.sub(r'^Hi (?!Commander)[^,\n]{1,40},', 'Hi Commander,', t)
        t = re.sub(r'^===== return to contents =====\n', '', t, flags=re.M | re.I)
        t = re.sub(r'^\{\{[^}]*transparent[^}]*\}\}\n', '', t, flags=re.M)  # Platzhalter-GIF
        # Werbe-Parameter (utm_…) aus Links entfernen
        t = re.sub(r'[?&]utm_[^|\]]*?(?=[|\]])', '', t)
        # Einzelkorrekturen
        for a, b in FIXES.get(nr, []):
            assert a in t, (nr, a[:60])
            t = t.replace(a, b)
        # Links auf andere Newsletter

        def nl_link(mm):
            url, text = mm.group(1), mm.group(2)
            i = re.search(r'id=([0-9a-f]{10})', url)
            if i and i.group(1) in by_id:
                target = by_id[i.group(1)]
                if target != nr:
                    return '[[NL:%s|%s]]' % (target, text)
            return mm.group(0)
        t = re.sub(r'\[\[https?://us2\.campaign-archive\d?\.com/home/[^|\]]*\|', '[[NL:start|', t)
        t = re.sub(r'\[\[(https?://us2\.campaign-archive[^|\]]*)\|([^\]]*)\]\]', nl_link, t)
        # Fettung um Links mit eigener Fettung: [[url|**x**]] -> **[[url|x]]**
        t = re.sub(r'\[\[([^|\]]+)\|\*\*([^\]]*?)\*\*\]\]', r'**[[\1|\2]]**', t)
        # E-Mail-Adresse nicht fett
        t = re.sub(r'\*\*([\w.]+@frontier\.co\.uk)\*\*', r'\1', t)
        t = re.sub(r' +([.,!?])', r'\1', t)
        t = re.sub(r'\]\]\*\*(?=[A-Za-z])', ']]** ', t)
        t = re.sub(r'(?<=[A-Za-z])\*\*\[\[', ' **[[', t)
        t = re.sub(r'^If you do not wish to receive any more.*\n', '', t, flags=re.M)
        t = re.sub(r'\n{3,}', '\n\n', t).strip() + '\n'
        open(os.path.join(outdir, 'en_%s.txt' % nr), 'w').write(t)


if __name__ == '__main__':
    main()