Strategy-as-Code/tools/parse_nap.py

#!/usr/bin/env python3
"""NAP2027 PDF → strukturēts JSON (starpposms; XML veido build_xml.py).

  python3 tools/parse_nap.py sources/nap2027/NAP2027.pdf build/nap2027.json

Lasa vārdus ar koordinātām (pdfplumber), noņem lapu galvenes un kājenes, atdala zemsvītras piezīmes,
atpazīst nodaļas, numurētos punktus [1]–[475], indikatoru un uzdevumu tabulas (ailes pēc galvenes koordinātām)
un pielikuma pamatojuma punktus.
"""
import json
import re
import sys
from collections import defaultdict

import pdfplumber

HEAD_TOP, FOOT_TOP = 52, 775
MARK = re.compile(r"^\[(\d{1,3})\]$")


def norm(s):
    s = re.sub(r"\s+", " ", s or "").strip()
    s = re.sub(r"\s+([,.;:)”])", r"\1", s)
    s = re.sub(r"([“(])\s+", r"\1", s)
    return repair(s)


from collections import Counter as _Counter
VOCAB = _Counter()


def load_vocab(pdf_path):
    """Word-form frequencies as pdftotext reads the body text — used to repair words split by kerning or cell hyphenation."""
    import subprocess
    raw = subprocess.run(["pdftotext", pdf_path, "-"], capture_output=True, text=True).stdout
    VOCAB.update(w.strip(".,;:()“”\"'") for w in raw.split())


def _better(whole, *parts):
    """Join when the whole word is attested more often than any of its fragments, or once while a fragment never is."""
    n = VOCAB.get(whole, 0) + VOCAB.get(whole.lower(), 0)
    return (n >= 2 and any(VOCAB.get(p, 0) < n for p in parts)) or (n >= 1 and any(VOCAB.get(p, 0) == 0 for p in parts))


def repair(s):
    if not s or not VOCAB:
        return s
    toks, out = s.split(" "), []
    for t in toks:
        if out:
            a, core = out[-1], t.rstrip(".,;:)”")
            ca = a.lstrip("“(")
            if "http" in a or "www." in a or ("/" in a and re.search(r"[_.%=?&]|/.*/|-$", a)):
                out.append(t)
                continue
            # three fragments "fundamentāl a s"
            if len(out) >= 2 and len(a) <= 2 and a.isalpha() and core[:1].islower() and \
                    _better(out[-2].lstrip("“(") + a + core, out[-2].lstrip("“("), a, core):
                out.pop()
                out[-1] = out[-1] + a + t
                continue
            # "paš - valdības"
            if a == "-" and len(out) >= 2 and core[:1].islower() and _better(out[-2].lstrip("“(") + core, out[-2].lstrip("“("), core):
                out.pop()
                out[-1] = out[-1] + t
                continue
            # "aizsar- dzības"
            if ca.endswith("-") and len(ca) > 2 and ca[-2].islower() and core[:1].islower() and \
                    (_better(ca[:-1] + core, ca[:-1], core) or VOCAB.get(core, 0) == 0):
                out[-1] = a[:-1] + t
                continue
            # dropped capital "O glekļa", kerning "pašvaldī bas"
            if ca[-1:].isalpha() and core[:1].isalpha() and core[:1].islower() and \
                    (_better(ca + core, ca, core) or (len(ca) == 1 and ca.isupper() and VOCAB.get((ca + core).lower(), 0) >= 2) or
                     (len(ca) == 1 and ca.isupper() and VOCAB.get(ca + core, 0) >= 1 and VOCAB.get(core, 0) == 0)):
                out[-1] = a + t
                continue
        out.append(t)
    return " ".join(out)


def fix_urls(s):
    """A URL broken across lines: rejoin the piece after - / _ . = ? & %."""
    prev = None
    while prev != s:
        prev = s
        s = re.sub(r"((?:https?://|www\.)\S*[-/_.%=?&])\s+(?=[\w%])", r"\1", s)
    return s


def squash(s):
    return re.sub(r"\s+", "", s or "")


def page_lines(pdf):
    """-> list of lines: {page, top, x0, words:[...], text, size, bold}; footnotes separately."""
    out, notes = [], []
    for pi, p in enumerate(pdf.pages):
        ws = p.extract_words(extra_attrs=["size", "fontname"], keep_blank_chars=False)
        ws = [w for w in ws if HEAD_TOP < w["top"] < FOOT_TOP]
        # footnotes: small text at the bottom of the page
        marks = [w["top"] for w in ws if w["size"] < 6.5 and w["text"].isdigit() and w["x0"] < 90 and w["top"] > 550]
        fz = min(marks) - 1.5 if marks else 9999
        infoot = lambda w: w["size"] < 10.5 and w["top"] >= fz
        foot = [w for w in ws if infoot(w)]
        body = [w for w in ws if not infoot(w) and w["size"] >= 7.5]
        smalls = [w for w in ws if not infoot(w) and w["size"] < 7.5]
        if foot:
            fsub = [w for w in foot if w["size"] < 6.5 and w["x0"] >= 90]
            foot = [dict(w) for w in foot if not (w["size"] < 6.5 and w["x0"] >= 90)]
            for sw in fsub:
                near = [m for m in foot if abs(m["top"] - sw["top"]) < 8 and -0.5 <= sw["x0"] - m["x1"] < 3]
                if near:
                    m = min(near, key=lambda m: sw["x0"] - m["x1"])
                    m["text"] += sw["text"].translate(str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉"))
            foot.sort(key=lambda w: (w["top"], w["x0"]))
            rows_f = []
            for w in foot:
                if rows_f and abs(w["top"] - rows_f[-1][0]["top"]) < 2.5:
                    rows_f[-1].append(w)
                else:
                    rows_f.append([w])
            cur = None
            for r in rows_f:
                r.sort(key=lambda w: w["x0"])
                if r[0]["size"] < 6.5 and r[0]["text"].isdigit() and r[0]["x0"] < 90:
                    cur = {"page": pi + 1, "number": int(r[0]["text"]), "text": " ".join(w["text"] for w in r[1:])}
                    notes.append(cur)
                elif cur:
                    cur["text"] += " " + " ".join(w["text"] for w in r)
        # merge glyphs split by kerning on one line
        body.sort(key=lambda w: (round(w["top"] / 2.5), w["x0"]))
        merged = []
        for w in body:
            if merged and abs(merged[-1]["top"] - w["top"]) < 2.5 and 0 <= w["x0"] - merged[-1]["x1"] < 0.9:
                m = merged[-1]
                m["text"] += w["text"]
                m["x1"] = w["x1"]
                m["bold"] = m["bold"] and "Bold" in w["fontname"]
            else:
                merged.append(dict(text=w["text"], x0=w["x0"], x1=w["x1"], top=w["top"], size=w["size"], page=pi + 1,
                                   bold="Bold" in w["fontname"], italic="Italic" in w["fontname"]))
        split = []
        for w in merged:
            m = re.match(r"^(\[\d{1,3}\])(.+)$", w["text"])
            if m:
                split.append(dict(w, text=m.group(1), x1=w["x0"] + 20))
                split.append(dict(w, text=m.group(2), x0=w["x0"] + 20.5))
            else:
                split.append(w)
        merged = split
        # footnote numbers glued to a word at the same font size ("attīstībā16.")
        page_notes = {n["number"] for n in notes if n["page"] == pi + 1}
        for m_ in merged:
            g = re.match(r"^(.*[a-zāčēģīķļņšūž”)])(\d{1,2})([.,;:]?)$", m_["text"])
            if g and int(g.group(2)) in page_notes:
                m_["text"] = g.group(1) + g.group(3)
                m_.setdefault("refs", []).append(int(g.group(2)))
        # small glyphs: subscripts (CO₂) join the word; superscript digits are footnote references
        SUBS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉")
        for sw in smalls:
            near = [m for m in merged if abs(m["top"] - sw["top"]) < 8 and -0.5 <= sw["x0"] - m["x1"] < 3]
            if not near:
                continue
            m = min(near, key=lambda m: sw["x0"] - m["x1"])
            if sw["top"] > m["top"] + 1.5:
                m["text"] += sw["text"].translate(SUBS)
                m["x1"] = sw["x1"]
            elif sw["text"].isdigit():
                m.setdefault("refs", []).append(int(sw["text"]))
        rows = defaultdict(list)
        for w in merged:
            rows[round(w["top"] / 2.5)].append(w)
        keys = sorted(rows)
        # join rows whose tops are within 2.5 pt (rounding boundary)
        grouped = []
        for k in keys:
            if grouped and abs(rows[k][0]["top"] - grouped[-1][0]["top"]) < 2.6:
                grouped[-1].extend(rows[k])
            else:
                grouped.append(list(rows[k]))
        for g in grouped:
            g.sort(key=lambda w: w["x0"])
            out.append({"page": pi + 1, "top": min(w["top"] for w in g), "x0": g[0]["x0"], "words": g,
                        "text": " ".join(w["text"] for w in g), "size": max(w["size"] for w in g),
                        "bold": all(w["bold"] for w in g)})
    return out, notes


SECTION_HEADS = [
    ("introduction", "IEVADS"), ("vision", "VĪZIJA PAR LATVIJAS NĀKOTNI 2027. GADĀ"), ("framework", "NAP2027 IETVARS"),
    ("strategicGoals", "NAP2027 STRATĒĢISKIE MĒRĶI"), ("spatial", "NAP2027 telpiskās attīstības perspektīva"),
    ("implementation", "NAP2027 īstenošanas, finansēšanas, uzraudzības un novērtēšanas process"), ("annex", "Pielikums"),
]
SUB = {
    "PRIORITĀTESMĒRĶIS": "priorityGoal", "RĪCĪBASVIRZIENAMĒRĶIS": "actionLineGoal", "RĪCĪBASVIRZIENAMĒRĶI": "actionLineGoal",
    "Stratēģiskomērķuindikatori": "indicators", "Rīcībasvirzienamērķaindikatori": "indicators",
    "Rīcībasvirzienamērķuindikatori": "indicators", "Rīcībasvirzienauzdevumi": "tasks",
}
IND_COLS = ["no", "name", "unit", "baseYear", "baseValue", "target2024", "target2027", "source"]
TASK_COLS = ["no", "text", "responsible", "coResponsible", "funding", "indicators"]


def header_centers(lines, kind):
    """Column centres from the header words of one table (the lines between the table title and the first row)."""
    ws = [w for ln in lines for w in ln["words"]]
    def c(w):
        return (w["x0"] + w["x1"]) / 2
    def first(txt, n=0):
        hits = sorted([w for w in ws if w["text"].startswith(txt)], key=lambda w: w["x0"])
        return hits[n] if len(hits) > n else None
    if kind == "indicators":
        hs = [first("Nr"), first("Progresa") or first("Rādītājs") or first("rādītājs") or first("Indikators"), first("Mēr"), first("Bāzes", 0),
              first("Bāzes", 1), first("Mērķa", 0), first("Mērķa", 1), first("Datu")]
    else:
        hs = [first("Nr"), first("Uzdevums"), (first("Atbildīgā") or first("Atbildī")), (first("Līdzatbildīgās") or first("Līdz")), first("Finanšu"), first("Indikators")]
    if any(h is None for h in hs):
        return None
    return [c(h) for h in hs]


def gutters(words, centers, min_gap=2.0):
    """Column boundaries from the empty vertical strips between the words of one table page.
    Between two neighbouring header centres the widest empty strip is the gutter; without one, the midpoint."""
    iv = sorted((w["x0"], w["x1"]) for w in words)
    gaps, end = [], None
    for a, b in iv:
        if end is not None and a - end >= min_gap:
            gaps.append((end, a))
        end = b if end is None else max(end, b)
    bounds = []
    for i in range(len(centers) - 1):
        lo, hi = centers[i], centers[i + 1]
        cand = [(g[1] - g[0], (g[0] + g[1]) / 2) for g in gaps if lo < (g[0] + g[1]) / 2 < hi]
        bounds.append(max(cand)[1] if cand else (lo + hi) / 2)
    return bounds


def assign(words, centers, names, bounds=None):
    bounds = bounds or [(centers[i] + centers[i + 1]) / 2 for i in range(len(centers) - 1)]
    cells = {n: [] for n in names}
    for w in words:
        cx = (w["x0"] + w["x1"]) / 2
        i = sum(1 for b in bounds if cx > b)
        i = max(i, 1)  # the row marker is not among the words: nothing belongs to the "Nr." column
        cells[names[i]].append(w)
    return cells


def cell_text(ws):
    ws = sorted(ws, key=lambda w: (w.get("page", 0), round(w["top"]), w["x0"]))
    t = norm(" ".join(w["text"] for w in ws))
    return re.sub(r"(\d{4}/\d{1,3}) (\d)", r"\1\2", t)  # "2018/2 019" broken inside a narrow cell


def split_by_gap(ws, gap=17.0):
    """Indicator names in a task row are separated by an empty line."""
    ws = sorted(ws, key=lambda w: (w.get("page", 0), round(w["top"]), w["x0"]))
    groups, last_top, last_page = [], None, None
    for w in ws:
        if last_top is None or w["top"] - last_top > gap or w.get("page") != last_page:
            groups.append([])
        groups[-1].append(w)
        last_top, last_page = w["top"], w.get("page")
    return [norm(" ".join(x["text"] for x in g)) for g in groups if g]


def table_pages(lines):
    """First pass: (kind, page) → words of table rows (≈10 pt lines between a table title and body-size text)."""
    acc, mode, started, tid = defaultdict(list), None, False, 0
    for ln in lines:
        sq = squash(ln["text"])
        if sq in SUB and ln["x0"] < 100:
            mode = {"indicators": "indicator", "tasks": "task"}.get(SUB[sq])
            started = False
            tid += mode is not None
            continue
        if mode and ln["size"] >= 11:
            mode = None
        if mode and MARK.match(ln["words"][0]["text"]):
            started = True
        if mode and started and ln["size"] < 11 and not ln["text"].startswith("*"):
            acc[(tid, ln["page"])] += ln["words"]
    return acc


def parse(path):
    pdf = pdfplumber.open(path)
    load_vocab(path)
    lines, notes = page_lines(pdf)
    page_words = table_pages(lines)
    page_bounds = {}
    tid = 0
    sections, items, evidence = [], [], []
    sec = {"id": "front", "kind": "front", "title": "Titullapa un saīsinājumi", "parent": None}
    sections.append(sec)
    pr = rv = None
    npr = nrv = 0
    mode, centers, pending_goal = "text", None, None
    centers_by = {}
    area_buf = []
    cur = None          # current item being filled
    hdr_buf = []
    funding_buf = None
    note_open = False
    abbrs = []
    problem, problem_open, problem_words = None, False, []
    spatial_note, spatial_note_page = False, None
    i = 0
    stats = defaultdict(int)
    stats_pages = []

    def close():
        nonlocal cur
        if cur is None:
            return
        refs = sorted({r for w in cur.get("_words", []) for r in w.get("refs", [])})
        if refs:
            cur["footnoteRefs"] = refs
        if cur["kind"] in ("indicator", "task"):
            cols = IND_COLS if cur["kind"] == "indicator" else TASK_COLS
            ws, ctr = cur.pop("_words"), cur.pop("_centers")
            ws = [w for w in ws if not (w["text"] in ("(", "[") and w["x0"] < 125) and w["text"] != "["]
            tid_ = cur.pop("_tid")
            cells = {n: [] for n in cols}
            for pg in sorted({w["page"] for w in ws}):
                pw = [w for w in ws if w["page"] == pg]
                key = (tid_, pg)
                if key not in page_bounds:
                    page_bounds[key] = gutters(page_words.get(key) or pw, ctr)
                b = page_bounds[key]
                for k, v in assign(pw, ctr, cols, b).items():
                    cells[k] += v
            if cur["kind"] == "indicator":
                for k in cols[1:]:
                    cur[k] = cell_text(cells[k]) or None
            else:
                cur["text"] = cell_text(cells["text"])
                cur["responsible"] = cell_text(cells["responsible"]) or None
                cur["coResponsible"] = cell_text(cells["coResponsible"]) or None
                cur["funding"] = cell_text(cells["funding"]) or None
                cur["indicatorNames"] = split_by_gap(cells["indicators"])
        else:
            ws = cur.pop("_words")
            bold = [w for w in ws if w["bold"]]
            cur["text"] = norm(" ".join(w["text"] for w in ws))
            cur["boldLead"] = norm(" ".join(w["text"] for w in ws[:len(ws)] if w["bold"])) if bold else None
        if "_area" in cur:
            cur["area"] = cur.pop("_area")[0] if cur["_area"] else None
        items.append(cur)
        cur = None

    while i < len(lines):
        ln = lines[i]
        t, sq = ln["text"], squash(ln["text"])
        # ---------------------------------------------------------------- big headings
        if ln["size"] >= 13.5 and ln["page"] > 3:
            j, title = i + 1, t
            while j < len(lines) and lines[j]["size"] >= 13.5 and abs(lines[j]["size"] - ln["size"]) < 0.5 \
                    and lines[j]["page"] == ln["page"] and lines[j]["top"] - lines[j - 1]["top"] < 26:
                title += " " + lines[j]["text"]
                j += 1
            title = norm(title)
            close()
            if funding_buf:
                funding_buf = None
            m = re.match(r"^Prioritāte “(.+)”$", title)
            m2 = re.match(r"^Rīcības virziens “(.+)”$", title)
            if sec.get("kind") == "annex" or (sections and any(s["kind"] == "annex" for s in sections)):
                pass
            if m:
                npr += 1
                pr = {"id": f"pr{npr}", "kind": "priority", "title": m.group(1), "parent": None}
                sections.append(pr)
                sec, rv, mode = pr, None, "text"
            elif m2:
                nrv += 1
                rv = {"id": f"{pr['id']}.rv{nrv}", "kind": "actionLine", "title": m2.group(1), "parent": pr["id"], "funding": None}
                sections.append(rv)
                sec, mode = rv, "text"
            elif title == "NAP2027 prioritāšu pamatojuma avoti":
                mode = "annex"
            else:
                kind = next((k for k, h in SECTION_HEADS if squash(h) == squash(title)), None)
                if kind:
                    sec = {"id": kind, "kind": kind, "title": title, "parent": None}
                    sections.append(sec)
                    pr = rv = None
                    mode = "annex" if kind == "annex" else "text"
                elif ln["page"] > 4:
                    stats["unknown_heading"] += 1
            i = j
            continue
        # ---------------------------------------------------------------- annex: evidence points per action line
        if mode == "annex":
            m = re.match(r"^Prioritāte “(.+)”$", norm(t))
            m2 = re.match(r"^Rīcības virziens “(.+)”?$", norm(t))
            if m:
                pr = next((s for s in sections if s["kind"] == "priority" and squash(s["title"]) == squash(m.group(1))), None)
                rv = None
                problem, problem_open = None, False
                i += 1
                continue
            if m2 and ln["x0"] < 90:
                title = norm(t)
                while not title.endswith("”") and i + 1 < len(lines):
                    i += 1
                    title = norm(title + " " + lines[i]["text"])
                name = re.match(r"^Rīcības virziens “(.+)”$", title).group(1)
                rv = next((s for s in sections if s["kind"] == "actionLine" and squash(s["title"]) == squash(name)), None)
                if rv is None:
                    stats["annex_unknown_line"] += 1
                problem, problem_open = None, False
                i += 1
                continue
            m3 = re.match(r"^(\d{1,3})\.\s", t)
            if m3 and ln["x0"] < 90:
                problem_open = False
                evidence.append({"number": int(m3.group(1)), "section": (rv or pr or {}).get("id"),
                                 "_words": ln["words"][1:], "page": ln["page"], "problem": problem})
            elif ln["bold"] and ln["x0"] < 90:
                # bold problem statement that groups the following evidence points ("…:")
                if problem_open and problem:
                    problem, problem_words = norm(problem + " " + t), problem_words + ln["words"]
                else:
                    problem, problem_words = norm(t), list(ln["words"])
                problem_open = not t.rstrip().endswith(":")
            elif problem_open and ln["x0"] < 90:
                # unnumbered evidence paragraph: bold lead without ":" continues as body text
                evidence.append({"number": None, "section": (rv or pr or {}).get("id"), "_words": problem_words + ln["words"],
                                 "page": ln["page"], "problem": None})
                problem, problem_open, problem_words = None, False, []
            elif evidence:
                evidence[-1]["_words"] += ln["words"]
            i += 1
            continue
        # ---------------------------------------------------------------- sub-headings
        if sq in SUB and ln["x0"] < 100:
            close()
            kind = SUB[sq]
            if kind in ("priorityGoal", "actionLineGoal"):
                pending_goal, mode = kind, "text"
            else:
                mode, hdr_buf = kind, []
                tid += 1
                # header lines until the first row marker
                j = i + 1
                while j < len(lines) and not MARK.match(lines[j]["words"][0]["text"]):
                    hdr_buf.append(lines[j])
                    j += 1
                c = header_centers(hdr_buf, kind)
                if c:
                    centers_by[kind] = c
                else:
                    stats["header_fallback_" + kind] += 1
                    stats.setdefault("header_fallback_pages", []).append(ln["page"])
                centers = centers_by.get(kind)
                i = j
                continue
            i += 1
            continue
        # ---------------------------------------------------------------- indicative funding of an action line
        if sq.startswith("Rīcībasvirzienapasākumu") or funding_buf is not None:
            close()
            funding_buf = (funding_buf or "") + " " + t
            if "EUR" in t:
                m = re.search(r"apjoms\s+([\d\s,]+)\s*milj\.\s*EUR", norm(funding_buf))
                if rv is not None:
                    rv["funding"] = {"text": norm(funding_buf), "millionEur": m.group(1).replace(" ", "") if m else None}
                funding_buf = None
                mode = "text"
            i += 1
            continue
        # ---------------------------------------------------------------- spatial perspective: area | items
        if sec.get("kind") == "spatial" and (t.startswith("Pilsētu un lauku mijiedarbības dimensijas") or spatial_note):
            if MARK.search(t) or ln["page"] != spatial_note_page and spatial_note:
                spatial_note = False
            else:
                close()
                if not spatial_note:
                    sec.setdefault("notes", []).append({"area": area_buf[0] if area_buf else None, "text": norm(t)})
                    spatial_note, spatial_note_page = True, ln["page"]
                else:
                    sec["notes"][-1]["text"] = norm(sec["notes"][-1]["text"] + ("\n" if t.startswith(("–", "−")) else " ") + t)
                i += 1
                continue
        if sec.get("kind") == "spatial":
            mk_i = next((k for k, w in enumerate(ln["words"]) if MARK.match(w["text"])), None)
            if mk_i is not None and ln["words"][mk_i]["x0"] > 180:
                close()
                left = [w for w in ln["words"][:mk_i] if w["x0"] < 195]
                if left:
                    area_buf[:] = [norm(" ".join(w["text"] for w in left))]
                cur = {"number": int(MARK.match(ln["words"][mk_i]["text"]).group(1)), "kind": "spatialDirection",
                       "section": "spatial", "page": ln["page"], "_words": ln["words"][mk_i + 1:], "_area": area_buf}
                i += 1
                continue
            if cur is not None and cur.get("_area") is not None:
                left = [w for w in ln["words"] if w["x0"] < 195]
                right = [w for w in ln["words"] if w["x0"] >= 195]
                if left and not right and len(left) <= 5:
                    area_buf[0] = norm(area_buf[0] + " " + " ".join(w["text"] for w in left)) if area_buf else norm(" ".join(w["text"] for w in left))
                    i += 1
                    continue
                if left and right and right[0]["x0"] > 190 and len(left) <= 4:
                    area_buf[0] = norm(area_buf[0] + " " + " ".join(w["text"] for w in left))
                    cur["_words"] += right
                    i += 1
                    continue
        # ---------------------------------------------------------------- table notes ("* ...", 9 pt)
        if (t.startswith("*") and ln["size"] < 9.5) or (note_open and ln["size"] < 9.5 and not MARK.match(ln["words"][0]["text"])):
            if t.startswith("*"):
                close()
                sec.setdefault("notes", []).append(norm(t.lstrip("* ")))
            else:
                sec["notes"][-1] = norm(sec["notes"][-1] + " " + t)
            note_open = True
            i += 1
            continue
        note_open = False
        # ---------------------------------------------------------------- thematic sub-sections inside an action line
        if rv is not None and 11.5 < ln["size"] < 13.5 and t.startswith("“") and norm(t).endswith("”") and ln["x0"] < 90:
            close()
            n_th = sum(1 for x in sections if x.get("parent") == rv["id"]) + 1
            sec = {"id": f"{rv['id']}.t{n_th}", "kind": "theme", "title": norm(t).strip("“”").strip(), "parent": rv["id"]}
            sections.append(sec)
            mode = "text"
            i += 1
            continue
        # ---------------------------------------------------------------- numbered items
        first = ln["words"][0]
        mk = MARK.match(first["text"])
        in_table = mode in ("indicators", "tasks")
        if in_table and not mk and ln["size"] >= 11:
            # body-size text ends the table (table cells are ≈10 pt)
            if True:
                close()
                mode = "text"
                in_table = False
        if mk:
            close()
            n = int(mk.group(1))
            if in_table and ln["size"] < 11:
                kind = "indicator" if mode == "indicators" else "task"
                cur = {"number": n, "kind": kind, "section": (sec if sec.get("kind") == "theme" else (rv or pr or sec))["id"], "page": ln["page"],
                       "_words": ln["words"][1:], "_centers": centers, "_tid": tid}
            else:
                if in_table:
                    mode = "text"
                k = "text"
                if pending_goal:
                    k = pending_goal
                cur = {"number": n, "kind": k, "section": (sec if sec.get("kind") == "theme" else (rv or pr or sec))["id"], "page": ln["page"], "_words": ln["words"][1:]}
                if pending_goal:
                    pending_goal = None if pending_goal == "priorityGoal" else pending_goal
            i += 1
            continue
        if cur is not None:
            cur["_words"] += [w for w in ln["words"] if not (w["text"] == "[" )]
        elif sec["kind"] == "front" and ln["page"] in (3, 4) and t != "IZMANTOTIE SAĪSINĀJUMI":
            m = re.match(r"^(.+?)\s+–\s+(.+)$", t)
            if m and ln["x0"] < 90 and len(m.group(1)) <= 40:
                abbrs.append({"abbr": m.group(1).strip(), "meaning": m.group(2).strip()})
            elif abbrs:
                abbrs[-1]["meaning"] = norm(abbrs[-1]["meaning"] + " " + t)
        elif sec["kind"] == "front":
            pass  # title page and table of contents
        else:
            stats["orphan_line"] += 1
            stats.setdefault("orphans", []).append([ln["page"], (sec or {}).get("id"), " ".join(w["text"] for w in ln["words"])[:110]])
        i += 1
    close()

    # goals: after RĪCĪBAS VIRZIENA MĒRĶIS(-I) only bold items are goals; the first non-bold item is context
    for it in items:
        if it["kind"] == "actionLineGoal" and not (it.get("boldLead") and len(it["boldLead"]) >= 0.6 * len(it["text"])):
            it["kind"] = "text"
    # once a non-goal follows, later items in the section are context
    seen_text = set()
    for it in items:
        if it["kind"] == "text":
            seen_text.add(it["section"])
        elif it["kind"] == "actionLineGoal" and it["section"] in seen_text:
            it["kind"] = "text"
    # strategic goals: paragraphs with a bold lead in the strategic goals section
    for it in items:
        if it["section"] == "strategicGoals" and it["kind"] == "text" and it.get("boldLead"):
            it["kind"] = "strategicGoal"
            it["title"] = it["boldLead"]
    for e in evidence:
        refs = sorted({r for w in e["_words"] for r in w.get("refs", [])})
        if refs:
            e["footnoteRefs"] = refs
        e["text"] = norm(" ".join(w["text"] for w in e.pop("_words")))
        urls = re.findall(r"(?:https?://|www\.)\S+", re.sub(r"(?<=[-/_.%=?&])\s+(?=\S)", "", e["text"]))
        clean = []
        for u in urls:
            u = u.rstrip(".,;")
            while u.endswith(")") and u.count(")") > u.count("("):
                u = u[:-1]
            clean.append(u.rstrip(".,;"))
        e["urls"] = clean
    return {"abbreviations": abbrs, "sections": sections, "items": items, "evidence": evidence, "footnotes": [dict(n, text=fix_urls(norm(n["text"]))) for n in notes], "stats": dict(stats)}


if __name__ == "__main__":
    doc = parse(sys.argv[1])
    out = sys.argv[2]
    with open(out, "w", encoding="utf-8") as f:
        json.dump(doc, f, ensure_ascii=False, indent=1)
    from collections import Counter
    print("sections", Counter(s["kind"] for s in doc["sections"]))
    print("items", len(doc["items"]), Counter(i["kind"] for i in doc["items"]))
    print("evidence", len(doc["evidence"]), "footnotes", len(doc["footnotes"]), "stats", doc["stats"])