"""Count the errors the MQL5 Market validator would count, per Strategy Tester run.

Reads the newest tester agent log (MetaTrader writes it as UTF-16LE), splits it into runs
at each "testing of ... started" line and prints, for every run: the EA, symbol and
timeframe, the number of deals, the final balance and the error lines grouped by message.

Usage:
    python count_tester_errors.py                 # all runs in today's newest agent log
    python count_tester_errors.py --since 14:05   # only runs started at or after 14:05
    python count_tester_errors.py --tester "D:\\MT5Data\\Tester\\<id>"
"""
import argparse
import collections
import glob
import os
import re

BAD = re.compile(r"failed|invalid|\[market closed\]|not enough money|no trading operations|stop out|error",
                 re.I)
IGNORE = ("CUDA enumeration",)   # driver notices, not trading errors


def newest_log(tester_dir):
    logs = glob.glob(os.path.join(tester_dir, "**", "Agent-*", "logs", "*.log"), recursive=True)
    if not logs:
        raise SystemExit(f"no agent logs under {tester_dir}")
    return max(logs, key=os.path.getmtime)


def read_runs(path, since):
    runs, cur = [], None
    text = open(path, "rb").read().decode("utf-16-le", "ignore")
    for line in text.splitlines():
        f = line.split("\t")
        if len(f) < 5:
            continue
        t, msg = f[2][:8], f[-1]
        if t < since:
            continue
        if "testing of" in msg and "started with inputs" in msg:
            m = re.match(r"(.*?):? testing of (\S+)", msg)       # "XAUUSD,D1: testing of Experts\x.ex5 ..."
            what = f"{m.group(1)} {m.group(2)}" if m else msg[:80]
            cur = dict(t=t, what=what, deals=0, balance="", bad=collections.Counter())
            runs.append(cur)
        elif cur is None:
            continue
        elif "final balance" in msg:
            cur["balance"] = msg.split("final balance")[1].strip()
        elif re.search(r"deal #\d+ (buy|sell)", msg):
            cur["deals"] += 1
        elif BAD.search(msg) and not any(s in msg for s in IGNORE):
            cur["bad"][re.sub(r"[\d.:#]+", "#", msg)[:100]] += 1
    return runs


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--tester", default=os.path.join(os.environ.get("APPDATA", ""), "MetaQuotes", "Tester"),
                    help="Tester data folder (default: %%APPDATA%%\\MetaQuotes\\Tester)")
    ap.add_argument("--since", default="00:00:00", help="only runs started at or after HH:MM[:SS]")
    a = ap.parse_args()
    since = a.since if a.since.count(":") == 2 else a.since + ":00"
    log = newest_log(a.tester)
    print("log:", log)
    for r in read_runs(log, since):
        total = sum(r["bad"].values())
        print(f"{r['t']}  {r['what']}: deals {r['deals']}, final balance {r['balance']}, error lines {total}")
        for msg, n in r["bad"].most_common(6):
            print(f"      {n:7d} x {msg}")


if __name__ == "__main__":
    main()