#!/usr/bin/env python3
"""Mailchimp-Newsletter (Wayback-Kopie) -> DokuWiki-Rohtext (Englisch) + Bildliste.

Aufruf: html2wiki.py <newsletter.html> <nr> <ausgabeordner>
Schreibt <ausgabe>/en_<nr>.txt (Seitenrumpf ohne Rahmen) und <ausgabe>/images_<nr>.json
"""
import json
import os
import re
import sys
import urllib.parse

from bs4 import BeautifulSoup, NavigableString, Tag

SKIP_IMG = re.compile(r'(social|google|tumblr|like_|cdn-images|facebook|twitter)', re.I)
WAYBACK = re.compile(r'^https?://web\.archive\.org/web/\d+[a-z_]*/', re.I)
MEDIA_NS = 'news:newsletter:{nr}:'


def clean_name(fname):
    """Wayback-Anhang entfernen, DokuWiki-taugliche Kleinschreibung."""
    base, ext = os.path.splitext(fname)
    base = re.sub(r'_[A-Za-z0-9]{4}$', '', base)  # _0oNk usw.
    base = re.sub(r'[^A-Za-z0-9]+', '_', base).strip('_').lower()
    ext = ext.lower().replace('jpeg', 'jpg')
    return base + ext


def wrap(text, mk):
    """Formatierung je Absatz/Zeile, Leerraum außerhalb der Marker."""
    parts = re.split(r'(\n+)', text)
    res = []
    for p in parts:
        if not p.strip() or p.startswith('\n'):
            res.append(p)
            continue
        lead = p[:len(p) - len(p.lstrip())]
        trail = p[len(p.rstrip()):]
        res.append(lead + mk + p.strip() + mk + trail)
    return ''.join(res)


def esc(text):
    """Sternchen-Folgen (z. B. Teaser 'the *****') vor DokuWiki schützen."""
    return re.sub(r'(?<!\*)(\*{3,})(?!\*)', r'%%\1%%', text)


def orig_url(href):
    href = WAYBACK.sub('', href or '')
    return href


class Conv:
    def __init__(self, path, nr):
        self.path = path
        self.dir = os.path.dirname(path)
        self.nr = nr
        self.out = []          # Liste von Blöcken
        self.inline = []       # laufender Absatz
        self.images = []
        self.seen = set()

    # ---------- Ausgabe
    def flush(self):
        txt = ''.join(self.inline)
        txt = re.sub(r'[ \t ]+', ' ', txt)
        txt = re.sub(r' *\n *', '\n', txt).strip()
        # leere Formatierungen entfernen
        prot = re.findall(r'%%.*?%%', txt)
        txt = re.sub(r'%%.*?%%', '\x00', txt)
        txt = txt.strip()
        for pr in prot:
            txt = txt.replace('\x00', pr, 1)
        if txt:
            for para in re.split(r'\n{2,}', txt):
                para = para.strip()
                if para:
                    self.out.append(('p', para.replace('\n', ' \\\\ ')))
        self.inline = []

    def add_img(self, img):
        src = urllib.parse.unquote(img.get('src', ''))
        if SKIP_IMG.search(src) or not src:
            return
        local = os.path.join(self.dir, src)
        if not os.path.exists(local):
            local = os.path.join(self.dir, os.path.basename(os.path.dirname(src)), os.path.basename(src))
        name = clean_name(os.path.basename(src))
        if name in self.seen:
            return
        self.seen.add(name)
        try:
            w = int(re.search(r'\d+', img.get('width') or '560').group())
        except Exception:
            w = 560
        link = img.parent.get('href') if img.parent and img.parent.name == 'a' else ''
        self.images.append({'name': name, 'local': local, 'width': w,
                            'orig': orig_url(link) if link and re.search(r'\.(jpe?g|png|gif)', link, re.I) else ''})
        self.flush()
        self.out.append(('img', name, w))

    # ---------- Durchlauf
    def walk(self, node, fmt=''):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                if ch.__class__.__name__ == 'Comment':
                    continue
                t = esc(str(ch).replace('\r', '').replace('\n', ' '))
                self.inline.append(t)
                continue
            if not isinstance(ch, Tag):
                continue
            n = ch.name
            if n in ('script', 'style'):
                continue
            if n == 'img':
                self.add_img(ch)
            elif n == 'br':
                self.inline.append('\n')
            elif n in ('h1', 'h2', 'h3', 'h4'):
                self.flush()
                t = re.sub(r'\s+', ' ', ch.get_text()).strip()
                if t:
                    self.out.append(('h', t))
                for im in ch.find_all('img'):
                    self.add_img(im)
            elif n in ('ul', 'ol'):
                self.flush()
                items = []
                for li in ch.find_all('li', recursive=False):
                    sub = Conv(self.path, self.nr)
                    sub.seen = self.seen
                    sub.walk(li)
                    sub.flush()
                    items.append(' '.join(b[1] for b in sub.out if b[0] == 'p'))
                    self.images += sub.images
                self.out.append(('ul' if n == 'ul' else 'ol', items))
            elif n in ('p', 'div', 'td', 'tr', 'table', 'tbody', 'blockquote', 'center'):
                self.flush()
                self.walk(ch)
                self.flush()
            elif n in ('strong', 'b', 'em', 'i'):
                mk = '**' if n in ('strong', 'b') else '//'
                inner = self.sub_inline(ch)
                self.inline.append(wrap(inner, mk))
            elif n == 'a':
                href = ch.get('href', '')
                if ch.find('img'):
                    for im in ch.find_all('img'):
                        self.add_img(im)
                    continue
                text = self.sub_inline(ch)
                if not href or href.startswith('#') or 'return to contents' in text.lower():
                    if 'return to contents' not in text.lower():
                        self.inline.append(text)
                    continue
                url = orig_url(href)
                if url.startswith('mailto:'):
                    self.inline.append(text)
                    continue
                self.inline.append('[[%s|%s]]' % (url, text.strip()) if text.strip() else '')
            else:
                self.walk(ch)

    def sub_inline(self, node):
        sub = Conv(self.path, self.nr)
        sub.seen = self.seen
        sub.walk(node)
        self.images += sub.images
        paras = []
        for b in sub.out:  # Bilder in Formatierung
            if b[0] == 'img':
                self.flush()
                self.out.append(b)
            elif b[0] == 'p':
                paras.append(b[1].replace(' \\\\ ', '\n'))
        paras.append(''.join(sub.inline))
        return re.sub(r'[ \t\u00a0]+', ' ', '\n\n'.join(x for x in paras if x.strip()))


def main():
    path, nr, outdir = sys.argv[1], sys.argv[2], sys.argv[3]
    html = open(path, encoding='utf-8', errors='replace').read()
    html = re.sub(r'<!-- BEGIN WAYBACK TOOLBAR INSERT -->.*?<!-- END WAYBACK TOOLBAR INSERT -->', '', html, flags=re.S)
    soup = BeautifulSoup(html, 'html.parser')
    body = soup.find(id='templateBody') or soup.body
    header = soup.find(id='templateHeader')
    c = Conv(path, nr)
    if header:
        for im in header.find_all('img'):
            c.add_img(im)
    c.walk(body)
    c.flush()
    # Ende abschneiden: "Follow Elite: Dangerous" und alles danach
    blocks = []
    for b in c.out:
        if b[0] == 'h' and re.match(r'follow elite', b[1], re.I):
            break
        blocks.append(b)
    ns = MEDIA_NS.format(nr=nr)
    lines = []
    # Kopfbild (mail_head / erstes Bild im Kopf) nicht in den Text, es ist das Box-Bild
    if blocks and blocks[0][0] == 'img':
        blocks = blocks[1:]
    grouped = []
    for b in blocks:
        if b[0] == 'img' and b[2] < 500 and grouped and grouped[-1][0] == 'imgs':
            grouped[-1][1].append(b)
        elif b[0] == 'img' and b[2] < 500:
            grouped.append(('imgs', [b]))
        else:
            grouped.append(b)
    blocks = grouped
    for b in blocks:
        if b[0] == 'imgs':
            lines.append(' '.join('{{%s%s?%d|}}' % (ns, i[1], i[2]) for i in b[1]) + '\n')
            continue
        if b[0] == 'p':
            lines.append(b[1] + '\n')
        elif b[0] == 'h':
            lines.append('===== %s =====\n' % b[1])
        elif b[0] == 'img':
            lines.append('{{%s%s?%d|}}\n' % (ns, b[1], min(b[2], 560)))
        elif b[0] in ('ul', 'ol'):
            mark = '*' if b[0] == 'ul' else '-'
            lines.append('\n'.join('  %s %s' % (mark, i) for i in b[1]) + '\n')
    os.makedirs(outdir, exist_ok=True)
    open(os.path.join(outdir, 'en_%s.txt' % nr), 'w').write('\n'.join(lines))
    json.dump(c.images, open(os.path.join(outdir, 'images_%s.json' % nr), 'w'), indent=1)


if __name__ == '__main__':
    main()