#!/usr/bin/env python3
"""
ANES external-efficacy extract for The Statecraft Blueprint, Obs 7.

Run this locally against the ANES Time Series Cumulative Data File CSV.
It reads only the five columns we need (so the 163MB file streams fine and
never has to move through the session bridge), and writes one small CSV.

    python3 anes_efficacy_extract.py "anes_timeseries_cdf_csv_20260205.csv" obs07_efficacy.csv

Then send me obs07_efficacy.csv — it will be a few KB.

Columns pulled
--------------
VCF0004   study year
VCF0009z  cross-year weight, combined face-to-face + web sample
VCF0609   "public officials don't care much what people like me think"
            1 = Agree   2 = Disagree   3 = Neither (1988+)   9 = DK/refused   0 = NA
VCF0613   "people like me don't have any say about what government does"
            same code frame — the second half of ANES's efficacy pair
VCF0648   ANES's own external efficacy index, 0-100, built from VCF0609 + VCF0613
            (each recoded 1=0, 2=100, 3=50, then averaged over valid responses).
            NOTE: this one is ALREADY polarity-correct — higher = officials are
            seen as more responsive — so it falls without any inversion.

What comes out
--------------
One row per study year:
  year
  n_valid_0609, pct_agree, pct_disagree, pct_neither      <- weighted, denominator = {1,2,3}
  pct_agree_excl_neither                                   <- weighted, denominator = {1,2}
  n_valid_0648, efficacy_index_mean                        <- weighted mean of VCF0648

Both denominators are reported deliberately: that is the exact question the
current chart.html and sources.md disagree about, and seeing them side by side
settles it. If the two agree columns diverge sharply in 2002, that confirms the
2002 dip is a middle-option artifact rather than a post-9/11 rally effect.
"""

import csv
import sys
from collections import defaultdict

NEEDED = ["VCF0004", "VCF0009z", "VCF0609", "VCF0613", "VCF0648"]


def num(s):
    s = (s or "").strip()
    if not s:
        return None
    try:
        return float(s)
    except ValueError:
        return None


def main(src, dst):
    # weighted tallies: year -> code -> summed weight
    tally = defaultdict(lambda: defaultdict(float))
    counts = defaultdict(lambda: defaultdict(int))
    idx_w = defaultdict(float)     # year -> summed weight for VCF0648
    idx_wx = defaultdict(float)    # year -> summed weight * value
    idx_n = defaultdict(int)

    with open(src, newline="", encoding="utf-8", errors="replace") as f:
        r = csv.DictReader(f)
        missing = [c for c in NEEDED if c not in r.fieldnames]
        if missing:
            sys.exit(
                f"Column(s) not found in {src}: {', '.join(missing)}\n"
                f"First few columns present: {', '.join(r.fieldnames[:12])}"
            )

        for row in r:
            year = num(row["VCF0004"])
            w = num(row["VCF0009z"])
            if year is None or w is None or w <= 0:
                continue
            year = int(year)

            code = num(row["VCF0609"])
            if code in (1.0, 2.0, 3.0):
                tally[year][int(code)] += w
                counts[year][int(code)] += 1

            v = num(row["VCF0648"])
            if v is not None and 0.0 <= v <= 100.0:
                idx_w[year] += w
                idx_wx[year] += w * v
                idx_n[year] += 1

    years = sorted(set(tally) | set(idx_w))
    with open(dst, "w", newline="") as f:
        out = csv.writer(f)
        out.writerow([
            "year",
            "n_valid_0609", "pct_agree", "pct_disagree", "pct_neither",
            "pct_agree_excl_neither",
            "n_valid_0648", "efficacy_index_mean",
        ])
        for y in years:
            t = tally[y]
            denom_all = t[1] + t[2] + t[3]
            denom_ad = t[1] + t[2]
            n = counts[y][1] + counts[y][2] + counts[y][3]

            def pct(x, d):
                return round(100.0 * x / d, 2) if d > 0 else ""

            out.writerow([
                y,
                n or "",
                pct(t[1], denom_all),
                pct(t[2], denom_all),
                pct(t[3], denom_all),
                pct(t[1], denom_ad),
                idx_n[y] or "",
                round(idx_wx[y] / idx_w[y], 2) if idx_w[y] > 0 else "",
            ])

    print(f"Wrote {dst} — {len(years)} study years.")


if __name__ == "__main__":
    if len(sys.argv) != 3:
        sys.exit(__doc__)
    main(sys.argv[1], sys.argv[2])