"""cobol-input-validation-class-inspect-test-numval.py - what the edit program should write, worked out
without a COBOL compiler.

The COBOL program checks every field of an 80-byte payment feed, writes one line per error to errors.txt,
writes the clean records to accepted.dat and prints control totals. This companion copies the GnuCOBOL 3.2
runtime rules the program depends on, checks each copy against GnuCOBOL's own test suite and IBM's documented
examples, and only then applies them to the data file.

Steps, in order:
  1. Self-check. Each rule is ported from the GnuCOBOL 3.2 release (gnucobol-3.2.tar.gz, 2023-07-28):
       TEST-NUMVAL          libcob/intrinsic.c cob_check_numval: the 32 cases in tests/testsuite.src/
                            run_functions.at, and IBM Enterprise COBOL 6.4 Language Reference p. 651
       NUMVAL               libcob/intrinsic.c numval: run_functions.at and the example in the 3.2 NEWS file
       TEST-DATE-YYYYMMDD   libcob/intrinsic.c: run_functions.at and the five examples on IBM LR p. 645
       CLASS                cobc/codegen.c output_class_name_definition (true only if every byte is in the
                            set; a numeric literal is an ordinal position): run_fundamental.at
       ALPHABETIC-UPPER     libcob/common.c cob_is_upper (upper-case letters or spaces)
       INSPECT TALLYING     libcob/strings.c (counts are added to the counter; a position counted by one
                            phrase is not counted by a later one): run_misc.at, the guide's 7.8.24 example
                            and the examples on IBM LR pp. 356-358
       LINE SEQUENTIAL      libcob/fileio.c lineseq_read and lineseq_size, with the defaults in
                            libcob/common.c (a short line is padded with spaces; a long line comes back in
                            pieces, the first with status 06, or truncated with status 04 when COB_LS_SPLIT
                            is false; a control character gives status 09; WRITE drops trailing spaces)
  2. Layout. Reads the copybook and works out every field's offset from its PICTURE (80 bytes). Reads the
     CLASS clauses, the 19 error texts, the amount limits and the output record out of the .cob, and the
     88-level values out of the copybook, so the expected output cannot drift from the listing.
  3. Expected output. Applies the program's checks to the data file, prints the console totals, errors.txt
     and accepted.dat, and asserts the result for every record against the faults the file was built with.
  4. Side notes used in the article: what 3.2's NUMVAL returns for the bad amounts when nothing tests them
     first, and the EBCDIC code points that "A" THRU "Z" and "a" THRU "z" would take in on z/OS, from
     Table 81 of the IBM reference (CCSID 1140), cross-checked with Python's cp1140 codec.

Optional:  python code/cobol-input-validation-class-inspect-test-numval.py errors.txt accepted.dat
           compares the files a real run wrote with the expected lines (trailing spaces ignored).
Standard library only; Python 3.8 or later. Run from the site folder.
"""
from decimal import Decimal
import codecs
import re
import sys

SLUG = "cobol-input-validation-class-inspect-test-numval"
CPY = f"code/{SLUG}.cpy"
COB = "code/cobol-input-validation.cob"
DAT = f"datasets/{SLUG}.dat"
COB_MAX_DIGITS = 38          # libcob/common.h in 3.2
CHECKS = 0


def check(cond, msg):
    global CHECKS
    assert cond, msg
    CHECKS += 1


def is_digit(c):
    return 48 <= c <= 57


# ------------------------------------------------------------- GnuCOBOL 3.2 rules ---
def test_numval(s: bytes) -> int:
    """cob_check_numval (srcfield, NULL, 0, 0): 0 if NUMVAL would accept s, else the position of the first
    character in error, or the length plus one when the string ends without a digit."""
    max_pos = len(s)
    if not max_pos:
        return 1
    plus_minus = False
    n = 0
    while n < max_pos:                      # leading positions
        c = s[n]
        if is_digit(c):
            break
        if c == 0x20:
            n += 1
            continue
        if c in b"+-":
            if plus_minus:
                return n + 1
            plus_minus = True
            n += 1
            continue
        if c in b",.":
            if c != ord("."):
                return n + 1
            break
        return n + 1
    if n == max_pos:
        return max_pos + 1
    digits, decimal_seen, space_seen = 0, False, False
    while n < max_pos:                      # the number itself
        c = s[n]
        if is_digit(c):
            digits += 1
            if digits > COB_MAX_DIGITS or space_seen:
                return n + 1
            n += 1
            continue
        if c in b",.":
            if decimal_seen or space_seen:
                return n + 1
            if c != ord("."):
                return n + 1                # a comma, and this is not NUMVAL-C
            decimal_seen = True
            if digits:
                if not is_digit(s[n - 1]):
                    return n + 1
            elif n < max_pos - 1 and not is_digit(s[n + 1]):
                return n + 2
            n += 1
            continue
        if c == 0x20:
            space_seen = True
            n += 1
            continue
        if c in b"+-":
            if plus_minus:
                return n + 1
            n += 1
            break
        if c in b"CD":
            if plus_minus:
                return n + 1
            if n < max_pos - 1 and s[n + 1] == (ord("R") if c == ord("C") else ord("B")):
                n += 2
                break
            return n + 2
        return n + 1
    if not digits:
        return max_pos + 1
    while n < max_pos:                      # only spaces may follow
        if s[n] != 0x20:
            return n + 1
        n += 1
    return 0


def numval(s: bytes):
    """numval (NUMVAL type) in 3.2: invalid characters are skipped and flagged, not refused.
    Returns (value, exception raised)."""
    if not s:
        return Decimal(0), False
    p, p_end = 0, len(s) - 1
    while p != p_end and s[p_end] in (0x20, 0x00):
        p_end -= 1
    while p != p_end and s[p] in (0x20, ord("0")):
        p += 1
    sign, digits, dec_digits, decimal_seen, exc = 0, "", 0, False, False
    i = p
    while i <= p_end:
        if p_end - i + 1 >= 2 and s[i:i + 2].upper() in (b"CR", b"DB"):
            if sign:
                exc = True
            else:
                i += 2
                while i <= p_end:
                    if s[i] != 0x20:
                        exc = True
                        break
                    i += 1
            sign = -1
            break
        c = s[i]
        i += 1
        if c == ord("0") and not digits and not decimal_seen:
            continue
        if is_digit(c):
            if decimal_seen:
                dec_digits += 1
            digits += chr(c)
            if len(digits) > COB_MAX_DIGITS:
                exc = True
                break
        elif c in b"+-":
            if sign:
                exc = True
            else:
                sign = 1 if c == ord("+") else -1
        elif c == 0x20:
            continue
        elif c == ord("."):
            if decimal_seen:
                exc = True
            decimal_seen = True
        else:
            exc = True
    value = Decimal(int(digits)).scaleb(-dec_digits) if digits else Decimal(0)
    if sign == -1 and digits:
        value = -value
    return value, exc


def leap(y):
    return (y % 4 == 0 and y % 100 != 0) or y % 400 == 0


MONTH_DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]


def test_date_yyyymmdd(n: int) -> int:
    """cob_intr_test_date_yyyymmdd: 0 valid, 1 year not 1601-9999, 2 month not 1-12, 3 day not in month."""
    year, rest = divmod(n, 10000)
    month, day = divmod(rest, 100)
    if not 1601 <= year <= 9999:
        return 1
    if not 1 <= month <= 12:
        return 2
    last = 29 if month == 2 and leap(year) else MONTH_DAYS[month]
    return 0 if 1 <= day <= last else 3


def display_get_int(data: bytes) -> int:
    """cob_display_get_int (libcob/move.c) for an unsigned PIC 9(n): only the low half-byte of each character
    is used (COB_D2I), so a space reads as 0 - the 3.2 NEWS entry on invalid numeric DISPLAY data."""
    val = 0
    for b in data:
        val = val * 10 + (b & 0x0F)
    return val


def class_set(items):
    """cobc's class table: a one-character literal is that byte, a THRU pair is the range between them,
    a numeric literal n is the character at ordinal position n (byte n - 1)."""
    out = set()
    for it in items:
        if isinstance(it, tuple):
            lo, hi = it
            out.update(range(min(lo, hi), max(lo, hi) + 1))
        else:
            out.add(it)
    return out


def in_class(data: bytes, cset) -> bool:
    return all(b in cset for b in data)


def is_upper(data: bytes) -> bool:
    """cob_is_upper: every byte a space or an upper-case letter (C locale)."""
    return all(b == 0x20 or 65 <= b <= 90 for b in data)


def is_numeric_x(data: bytes) -> bool:
    """NUMERIC on an alphanumeric or unsigned DISPLAY item: digits only."""
    return all(is_digit(b) for b in data)


class Inspect:
    """INSPECT ... TALLYING, as libcob/strings.c runs it: one pass per phrase over its region, a mark on every
    position counted so a later phrase skips it, and every count ADDED to its counter."""

    def __init__(self, data: bytes):
        self.data = data
        self.mark = [False] * len(data)

    def region(self, before=None, after=None):
        start, end = 0, len(self.data)
        if before is not None:
            k = self.data.find(before, start, end)
            if k >= 0:
                end = k
        if after is not None:
            k = self.data.find(after, start, end)
            start = k + len(after) if k >= 0 else end
        return start, end

    def characters(self, before=None, after=None):
        s, e = self.region(before, after)
        n = sum(1 for i in range(s, e) if not self.mark[i])
        for i in range(s, e):
            self.mark[i] = True
        return n

    def all(self, what: bytes, before=None, after=None):
        s, e = self.region(before, after)
        n, i = 0, s
        while i <= e - len(what):
            if self.data[i:i + len(what)] == what and not any(self.mark[i:i + len(what)]):
                n += 1
                for j in range(i, i + len(what)):
                    self.mark[j] = True
                i += len(what)
            else:
                i += 1
        return n

    def leading(self, what: bytes, before=None, after=None):
        s, e = self.region(before, after)
        n, i = 0, s
        while i + len(what) <= e and self.data[i:i + len(what)] == what:
            if not any(self.mark[i:i + len(what)]):
                n += 1
                for j in range(i, i + len(what)):
                    self.mark[j] = True
            i += len(what)
        return n


BAD_CHAR_OK = {0x08, 0x1B, 0x0C, 0x0F, 0x09}     # BS, ESC, FF, SI, TAB pass the 3.2 read check


def lineseq_read(data: bytes, rec_len: int, split=True):
    """lineseq_read with COB_LS_VALIDATE true: [(record, status)], one entry per READ.
    split=True is the 3.2 default (COB_LS_SPLIT): a line longer than the record comes back in pieces, the
    first with status 06. split=False truncates it and sets status 04."""
    out, pos, end = [], 0, len(data)
    while pos < end:
        sts, buf = "00", bytearray()
        while pos < end:
            c = data[pos]
            pos += 1
            if c == 0x0A:
                break
            if c < 0x20 and c not in BAD_CHAR_OK:
                sts = "09"
            if len(buf) < rec_len:
                buf.append(c)
                if len(buf) == rec_len and split:
                    if pos < end and data[pos] == 0x0A:
                        pos += 1            # the line ends exactly here
                    else:
                        sts = "06"          # the rest of the line is the next READ
                    break
            else:
                sts = "04"
        out.append((bytes(buf.ljust(rec_len, b" ")), sts))
    return out


def take_records(reads):
    """Take-Record: the pieces after a status-06 read are dropped, so records are numbered by line."""
    out, in_long, line = [], False, 0
    for rec, sts in reads:
        check(sts in ("00", "04", "06", "09"), f"usable read status {sts}")
        if in_long:
            if sts != "06":
                in_long = False
            continue
        line += 1
        if sts == "06":
            in_long = True
        out.append((line, rec, sts))
    return out


def written(record: bytes) -> bytes:
    """lineseq_size: a LINE SEQUENTIAL WRITE drops the trailing spaces (COB_LS_FIXED is false by default)."""
    return record.rstrip(b" ")


# ------------------------------------------------------------------ 1. self-check ---
def self_check():
    # TEST-NUMVAL: tests/testsuite.src/run_functions.at, AT_SETUP([FUNCTION TEST-NUMVAL]), tests 1-32
    suite = [("+ 1", 0), (" + 1", 0), ("- 1", 0), (" - 1", 0), ("+- 1", 2), ("1 +", 0), ("1 -", 0),
             ("1 +-", 4), ("1 -+", 4), ("+ 1.1", 0), ("- 1.1", 0), ("1.1 +", 0), ("1.1 -", 0),
             ("1.1 CR", 0), ("1.1 DB", 0), ("1.1 -CR", 6), ("1.1 +DB", 6), ("1.1 CDB", 6),
             ("+1.1 CR", 6), ("+      ", 8), ("  0.   ", 0), ("  .0   ", 0), ("  . 0", 4), ("  0 .", 5),
             ("   0+0", 6), ("   0+  0", 8), ("1.1 D", 6), ("", 1), ("   ", 4), ("  .", 4), ("    ", 5),
             (" 1  T", 5)]
    check(len(suite) == 32, "32 TEST-NUMVAL cases")
    for text, want in suite:
        check(test_numval(text.encode()) == want, f"TEST-NUMVAL {text!r} -> {want}")
    # IBM 6.4 LR p. 651: '0 1' returns 3; ' +.' is incomplete, so the length plus one
    check(test_numval(b"0 1") == 3, "IBM: '0 1' -> 3")
    check(test_numval(b" +.") == 4, "IBM: ' +.' -> 4")

    # NUMVAL: run_functions.at AT_SETUP([FUNCTION NUMVAL]); BAD1 and BAD2 are the 3.2 results
    for text, want in [(" -9876.1234 ", "-9876.1234"), (" 19876.1234 CR".ljust(18), "-19876.1234"),
                       (" 19876.1234 cR".ljust(18), "-19876.1234"), ("-042.3240", "-42.324"),
                       ("+04232400", "4232400"), (" 00.00430", "0.0043"), (" " * 20, "0"),
                       (" 1. A-0B4.5".ljust(18), "-1.045"), ("+0@0%0=0*0!0&0^0)10-", "10")]:
        check(numval(text.encode())[0] == Decimal(want), f"NUMVAL {text!r} -> {want}")
    # the NEWS example: '"1. A-0B4.5" now returns -1.045', with the exception raised
    v, exc = numval(b"1. A-0B4.5")
    check(v == Decimal("-1.045") and exc, "NEWS: 1. A-0B4.5 -> -1.045, exception")
    check(numval(b"1250.75")[1] is False, "valid NUMVAL raises nothing")

    # TEST-DATE-YYYYMMDD: run_functions.at (20020231 -> 3) and IBM LR p. 645
    for n, want in [(20020231, 3), (19950215, 0), (12950215, 1), (912950215, 1), (19921415, 2),
                    (19950240, 3)]:
        check(test_date_yyyymmdd(n) == want, f"TEST-DATE-YYYYMMDD {n} -> {want}")
    # the Gregorian rule in valid_day_of_month / leap_year
    for n, want in [(20240229, 0), (20250229, 3), (20000229, 0), (19000229, 3), (16010101, 0),
                    (16001231, 1), (99991231, 0), (20261399, 2), (20260431, 3), (0, 1)]:
        check(test_date_yyyymmdd(n) == want, f"leap/limits {n} -> {want}")
    # why NUMERIC comes first: a date typed with a space for a zero reads as a valid number
    check(display_get_int(b"20261001") == 20261001, "digits read as themselves")
    check(display_get_int(b"2026 101") == 20260101, "a space reads as 0")
    check(not is_numeric_x(b"2026 101") and test_date_yyyymmdd(display_get_int(b"2026 101")) == 0,
          "2026 101 fails NUMERIC but would pass TEST-DATE-YYYYMMDD as 20260101")

    # CLASS: run_fundamental.at AT_SETUP([SPECIAL-NAMES CLASS]); EVEN is written as ordinals 49 51 53 55 57
    hexa = class_set([(48, 57), (65, 70)])
    odd = class_set([ord(c) for c in "13579"])
    even = class_set([n - 1 for n in (49, 51, 53, 55, 57)])
    check(in_class(b"3", hexa) and in_class(b"3", odd), "'3' is HEXA and ODD")
    check(in_class(b"2", even) and not in_class(b"3", even), "2 is EVEN, 3 is not")
    check(sorted(even) == [48, 50, 52, 54, 56], "ordinal n is byte n - 1")
    check(not in_class(b"3G", hexa), "every byte must be in the class")

    # ALPHABETIC-UPPER lets a space through; a class without a space does not
    check(is_upper(b"US ") and is_upper(b"USD") and not is_upper(b"usd"), "cob_is_upper")

    # INSPECT TALLYING: run_misc.at (TALLYING BEFORE / AFTER)
    check(Inspect(b"ABC ").characters(before=b" ") == 3, "ABC_ before space -> 3")
    check(Inspect(b" ABC").characters(before=b" ") == 0, "_ABC before space -> 0")
    check(Inspect(b"ABC ").characters(after=b" ") == 0, "ABC_ after space -> 0")
    check(Inspect(b" ABC").characters(after=b" ") == 3, "_ABC after space -> 3")
    # the guide, 7.8.24: --ABCDEF----BCDEF-- gives C-ABC 1 and C-BCDE 1
    ins = Inspect(b"--ABCDEF----BCDEF--")
    check((ins.all(b"ABC"), ins.all(b"BCDE")) == (1, 1), "guide 7.8.24 priority example")
    # IBM LR p. 356: 'In order to form a' has 4 spaces; p. 358: 14, 2 and 0
    check(Inspect(b"In order to form a").all(b" ") == 4, "IBM TALLYING example")
    ins = Inspect(b"$some.confusing_text with.hyphens-periods.spaces".ljust(50))
    got = (ins.characters(after=b"$", before=b"_"), ins.all(b".", after=b"h"), ins.all(b".", before=b"a"))
    check(got == (14, 2, 0), f"IBM BEFORE/AFTER example {got}")
    # the counter is never zeroed by INSPECT: the same statement twice doubles it
    counter = 0
    for _ in range(2):
        counter += Inspect(b"   42178").leading(b" ")
    check(counter == 6, "TALLYING adds to the count field")

    # LINE SEQUENTIAL read: pad a short line, split (06) or truncate (04) a long one, 09 for a control
    # character, a tab allowed, and a line of exactly the record length read whole
    sample = b"AB\n" + b"X" * 6 + b"\nA\x00\nA\tB\nABCDE\n"
    recs = lineseq_read(sample, 5)
    check(recs == [(b"AB   ", "00"), (b"XXXXX", "06"), (b"X    ", "00"), (b"A\x00   ", "09"),
                   (b"A\tB  ", "00"), (b"ABCDE", "00")], f"lineseq_read split {recs}")
    recs = lineseq_read(sample, 5, split=False)
    check(recs == [(b"AB   ", "00"), (b"XXXXX", "04"), (b"A\x00   ", "09"), (b"A\tB  ", "00"),
                   (b"ABCDE", "00")], f"lineseq_read no split {recs}")
    lines = [(n, s) for n, _, s in take_records(lineseq_read(sample, 5))]
    check(lines == [(1, "00"), (2, "06"), (3, "09"), (4, "00"), (5, "00")], f"take_records {lines}")


# ---------------------------------------------------------------------- 2. layout ---
def pic_size(pic):
    size = 0
    for sym, rep in re.findall(r"([X9ASVZ.,])(?:\((\d+)\))?", pic.upper()):
        n = int(rep) if rep else 1
        if sym not in "SV":
            size += n
    return size


def read_layout():
    fields, offset = {}, 0
    values88 = {}
    last = None
    text = open(CPY, encoding="ascii").read()
    longest = max(text.splitlines(), key=len)
    check(len(longest) <= 72, f"copybook lines end by column 72: {longest!r}")
    body = re.sub(r"\*>.*", "", text)
    for m in re.finditer(r"(\d\d)\s+([\w-]+)(\s+REDEFINES\s+[\w-]+)?\s+(?:PIC\s+(\S+?)\.|VALUE\s+(.+?)\.)",
                         body, flags=re.S):
        level, name, redefines, pic, value = m.groups()
        if level == "88":
            values88[name] = re.findall(r'"([^"]*)"', value)
            continue
        if level == "01":
            continue
        size = pic_size(pic)
        if redefines:
            fields[name] = (fields[last][0], size)
            continue
        fields[name] = (offset, size)
        offset += size
        last = name
    return fields, offset, values88


def read_listing():
    src = open(COB, encoding="ascii").read()
    m = re.search(r"SPECIAL-NAMES\.(.*?)INPUT-OUTPUT SECTION", src, flags=re.S)
    special = re.sub(r"\*>.*", "", m.group(1))
    classes = {}
    for cm in re.finditer(r"CLASS\s+([\w-]+)\s+IS\s+(.*?)(?=CLASS\s|\.\s*$)", special, flags=re.S):
        items, toks = [], re.findall(r'"[^"]"|THRU|\d+', cm.group(2))
        i = 0
        while i < len(toks):
            lo = ord(toks[i][1]) if toks[i].startswith('"') else int(toks[i]) - 1
            if i + 2 < len(toks) and toks[i + 1] == "THRU":
                hi = ord(toks[i + 2][1]) if toks[i + 2].startswith('"') else int(toks[i + 2]) - 1
                items.append((lo, hi))
                i += 3
            else:
                items.append(lo)
                i += 1
        classes[cm.group(1)] = class_set(items)
    texts = re.findall(r'05  FILLER PIC X\(28\) VALUE "([^"]*)"\.', src)
    limits = re.search(r"WS-Amount < ([\d.]+) OR WS-Amount > ([\d.]+)", src).groups()
    wide = re.search(r"WS-Amount-Wide\s+PIC S9\((\d+)\)V9\((\d+)\)\.", src).groups()
    narrow = re.search(r"WS-Amount\s+PIC S9\((\d+)\)V99\.", src).group(1)
    good = re.findall(r"05  (Good-[\w-]+)\s+PIC (\S+?)\.", src)
    err_line = re.search(r"01  Err-Line\.(.*?)01  WS-Count-Out", src, flags=re.S).group(1)
    el_sizes = [pic_size(p) for p in re.findall(r"PIC (\S+?)(?: VALUE|\.)", err_line)]
    record_x = int(re.search(r"01  Err-Record\s+PIC X\((\d+)\)\.", src).group(1))
    return dict(classes=classes, texts=texts, low=Decimal(limits[0]), high=Decimal(limits[1]),
                wide_int=int(wide[0]), wide_dec=int(wide[1]), narrow_int=int(narrow), good=good,
                el_sizes=el_sizes, record_x=record_x, src=src)


# ------------------------------------------------------------- 3. expected output ---
def validate(rec: bytes, sts: str, F, L, V):
    """The checks of Validate-Record, in the listing's order: [(error number, value shown, position)]."""
    def fld(name):
        off, size = F[name]
        return rec[off:off + size]

    errs = []
    if sts in ("04", "06"):
        return [(1, b"", 0)], None
    if sts == "09":
        return [(2, b"", 0)], None
    check(sts == "00", f"unexpected status {sts}")
    cls = L["classes"]

    acct = fld("In-Account")                                       # Check-Account
    if not is_numeric_x(acct):
        lead = Inspect(acct).leading(b" ")
        if lead == 0 or lead == len(acct):
            errs.append((3, acct, 0))
        elif is_numeric_x(acct[lead:]):
            errs.append((4, acct, 0))
        else:
            errs.append((3, acct, 0))
    name = fld("In-Name")                                          # Check-Name
    if name == b" " * len(name):
        errs.append((5, name, 0))
    elif not in_class(name[:1], cls["Letter"]) or not in_class(name, cls["Name-Char"]):
        errs.append((6, name, 0))
    cur = fld("In-Currency")                                       # Check-Currency
    if not in_class(cur, cls["Capital"]):
        errs.append((7, cur, 0))
    elif cur.decode("ascii") not in V["Currency-Accepted"]:
        errs.append((8, cur, 0))
    amt = fld("In-Amount")                                         # Check-Amount
    amount = None
    pos = test_numval(amt)
    if pos:
        errs.append((9, amt, pos))
    else:
        value, exc = numval(amt)
        check(not exc, "NUMVAL after a clean TEST-NUMVAL raises nothing")
        wide = value.quantize(Decimal(1).scaleb(-L["wide_dec"]), rounding="ROUND_DOWN")
        if abs(wide) >= Decimal(10) ** L["wide_int"]:
            errs.append((11, amt, 0))                              # ON SIZE ERROR
        else:
            amount = wide.quantize(Decimal("0.01"), rounding="ROUND_DOWN")   # MOVE truncates
            if amount != wide:
                errs.append((10, amt, 0))
            elif amount < L["low"] or amount > L["high"]:
                errs.append((11, amt, 0))
    date_x = fld("In-Value-Date-X")                                # Check-Date
    if not is_numeric_x(date_x):
        errs.append((12, date_x, 0))
    else:
        r = test_date_yyyymmdd(int(date_x))
        if r:
            errs.append((12 + r, date_x, 0))
    typ, ref = fld("In-Type"), fld("In-Reference")                 # Check-Type
    if typ.decode("ascii") not in V["Type-Valid"]:
        errs.append((16, typ, 0))
    elif typ.decode("ascii") in V["Type-Refund"] and ref == b" " * len(ref):
        errs.append((17, ref, 0))
    if ref != b" " * len(ref):                                     # Check-Reference
        ins = Inspect(ref)
        ref_len = ins.characters(before=b" ")
        ref_spaces = ins.all(b" ")
        if ref_len == 0 or ref_len + ref_spaces < len(ref):
            errs.append((18, ref, 0))
        elif not in_class(ref[:ref_len], cls["Ref-Char"]):
            errs.append((19, ref, 0))
    return errs, amount


def err_line(recno, no, value, pos, texts):
    shown = (b"[" + value + b"]").decode("ascii") if value else ""
    line = (f"{recno:>6}  E{no:02d}  {texts[no - 1]:<28}  {shown:<26} {'at' if pos else '  '} "
            f"{pos if pos else '':>2}")
    return line.rstrip()


def good_line(rec, amount, F):
    def fld(name):
        off, size = F[name]
        return rec[off:off + size]
    cents = int(amount * 100)
    out = (fld("In-Account") + fld("In-Name") + fld("In-Currency") + f"{cents:07d}".encode()
           + fld("In-Value-Date-X") + fld("In-Type") + fld("In-Reference"))
    return written(out).decode("ascii")


def edit_count(n):
    return f"{n:>7,}"


def edit_total(v):
    return f"{v:>14,.2f}"


# The faults the data file was built with: record number -> [(error number, TEST-NUMVAL position)]
DESIGN = {1: [], 2: [], 3: [(4, 0)], 4: [(3, 0), (6, 0), (16, 0)], 5: [(9, 2), (14, 0)],
          6: [(7, 0), (10, 0)], 7: [(11, 0), (15, 0)], 8: [(8, 0), (11, 0)], 9: [(7, 0), (16, 0)],
          10: [(17, 0)], 11: [(18, 0)], 12: [(19, 0)], 13: [(5, 0), (12, 0)], 14: [(13, 0)], 15: [],
          16: [(11, 0)], 17: [(9, 1)], 18: [(9, 5)], 19: [], 20: [(1, 0)], 21: [(2, 0)], 22: [],
          23: [(11, 0)], 24: []}


def main():
    self_check()
    print(f"self-check: {CHECKS} checks against the 3.2 test suite, the guide and IBM's examples")

    F, rec_len, V = read_layout()
    L = read_listing()
    check(rec_len == 80, f"copybook record is {rec_len} bytes")
    expect_off = {"In-Account": (0, 8), "In-Name": (8, 24), "In-Currency": (32, 3), "In-Amount": (35, 12),
                  "In-Value-Date-X": (47, 8), "In-Value-Date": (47, 8), "In-Type": (55, 1),
                  "In-Reference": (56, 16), "FILLER": (72, 8)}
    for name, where in expect_off.items():
        check(F.get(name) == where, f"{name} at {F.get(name)}")
    check(V == {"Currency-Accepted": ["USD", "EUR", "GBP"], "Type-Valid": ["P", "R"], "Type-Refund": ["R"]},
          f"88 levels {V}")
    check(len(L["texts"]) == 19 and all(len(t) <= 28 for t in L["texts"]), "19 error texts of <= 28")
    check(re.search(r"Err-Text\s+PIC X\(28\) OCCURS 19 TIMES", L["src"]) is not None, "table of 19")
    check(re.search(r"Code-Count\s+PIC 9\(4\) OCCURS 19 TIMES", L["src"]) is not None, "19 counters")
    check((L["low"], L["high"]) == (Decimal("0.01"), Decimal("50000.00")), "amount limits")
    check((L["wide_int"], L["wide_dec"], L["narrow_int"]) == (7, 11, 7), "wide S9(7)V9(11), narrow S9(7)V99")
    check(L["wide_int"] + L["wide_dec"] == 18, "the wide field is 18 digits")
    check(L["wide_dec"] >= F["In-Amount"][1] - 1, "the wide field keeps every decimal a 12-byte amount can hold")
    check(sum(L["el_sizes"]) == L["record_x"] == 75, f"error line {sum(L['el_sizes'])} bytes")
    good_size = sum(pic_size(p) for _, p in L["good"])
    check(good_size == 67, f"accepted record {good_size} bytes")
    letters = set(range(65, 91)) | set(range(97, 123))
    check(L["classes"]["Capital"] == set(range(65, 91)), "Capital is A-Z")
    check(L["classes"]["Letter"] == letters, "Letter is A-Z and a-z")
    check(L["classes"]["Name-Char"] == letters | {32, 39, 45, 46}, "Name-Char adds space ' - .")
    check(L["classes"]["Ref-Char"] == set(range(65, 91)) | set(range(48, 58)) | {45}, "Ref-Char A-Z 0-9 -")
    check(max(len(line) for line in L["src"].splitlines()) <= 255, "free-format lines within 255")

    data = open(DAT, "rb").read()
    check(b"\r" not in data and data.endswith(b"\n"), "LF line endings only")
    lines = data.split(b"\n")[:-1]
    check(len(lines) == 24, f"{len(lines)} lines")
    check([i for i, ln in enumerate(lines, 1) if len(ln) != 80] == [20], "every line 80 bytes but line 20")
    check(len(lines[19]) == 81, "line 20 is 81 bytes")
    check(lines[20][:8] == b"\x00" * 8, "line 21 has LOW-VALUES in the account")

    reads = lineseq_read(data, rec_len)
    check(len(reads) == 25, "25 READs: line 20 comes back in two pieces")
    check([s for _, s in reads].count("00") == 23, "23 reads with status 00, one of them the tail of line 20")
    check((reads[19][1], reads[20], reads[21][1]) == ("06", (b" " * 80, "00"), "09"),
          "line 20 gives 06, its tail is one space, line 21 gives 09")

    def run(split, design):
        errors, goods, counts = [], [], [0] * 19
        total, accepted, rejected = Decimal(0), 0, 0
        records = take_records(lineseq_read(data, rec_len, split))
        for recno, rec, sts in records:
            errs, amount = validate(rec, sts, F, L, V)
            if design:
                check([(n, p) for n, _, p in errs] == DESIGN[recno], f"record {recno}: {errs}")
            for no, value, pos in errs:
                counts[no - 1] += 1
                errors.append(err_line(recno, no, value, pos, L["texts"]))
            if errs:
                rejected += 1
            else:
                accepted += 1
                total += amount
                goods.append(good_line(rec, amount, F))
        return records, errors, goods, counts, total, accepted, rejected

    records, errors, goods, counts, total, accepted, rejected = run(split=True, design=True)
    other = run(split=False, design=False)
    check(other[1:] == (errors, goods, counts, total, accepted, rejected),
          "COB_LS_SPLIT false (status 04) gives the same output as the 3.2 default (status 06)")
    check([s for _, _, s in other[0]][19] == "04", "without splitting, line 20 reads with status 04")

    check(all(counts), "every one of the 19 rules fires at least once")
    check((len(records), accepted, rejected, len(errors)) == (24, 6, 18, 26), "24 read, 6 accepted, 18, 26")
    check(total == Decimal("53966.44"), f"accepted amount {total}")
    check(max(len(e) for e in errors) <= 75, "error lines fit the 75-byte record")
    check(errors[0] == "     3  E04  ACCOUNT NOT ZERO-FILLED       [   42178]", errors[0])
    check(errors[4] == "     5  E09  AMOUNT NOT A NUMBER" + " " * 11 + "[1,250.75    ]" + " " * 13 + "at  2",
          errors[4])

    console = [f"Records read     : {edit_count(len(records))}",
               f"Records accepted : {edit_count(accepted)}",
               f"Records rejected : {edit_count(rejected)}",
               f"Errors written   : {edit_count(len(errors))}",
               f"Accepted amount  : {edit_total(total)}",
               "Errors by code:"]
    console += [f"  E{i + 1:02d}{edit_count(c)}  {L['texts'][i]}" for i, c in enumerate(counts)]

    # ------------------------------------------------------------ 4. side notes ---
    guesses = {}
    for recno, rec, sts in records:
        off, size = F["In-Amount"]
        amt = rec[off:off + size]
        if sts == "00" and test_numval(amt):
            guesses[amt.decode().strip()] = numval(amt)
    check(guesses == {"1,250.75": (Decimal("1250.75"), True), "$45.00": (Decimal("45.00"), True),
                      "12.5.0": (Decimal("12.50"), True)}, f"NUMVAL guesses {guesses}")
    check(numval(b"19.999")[0] == Decimal("19.999"), "NUMVAL keeps the third decimal")
    check(is_upper(b"US ") and not in_class(b"US ", L["classes"]["Capital"]), "US_ passes only ALPHABETIC-UPPER")

    # IBM 6.4 LR Table 81 (CCSID 1140), pp. 751-754: where the letters sit in EBCDIC
    ebcdic = {"A-I": (0xC1, 0xC9), "J-R": (0xD1, 0xD9), "S-Z": (0xE2, 0xE9),
              "a-i": (0x81, 0x89), "j-r": (0x91, 0x99), "s-z": (0xA2, 0xA9)}
    named = {0xD0: "}", 0xE0: "\\", 0xC0: "{", 0xA1: "~", 0x9F: chr(0x20AC)}   # 0x20AC: euro sign
    for run, (lo, hi) in ebcdic.items():
        text = codecs.decode(bytes(range(lo, hi + 1)), "cp1140")
        first, last = run.split("-")
        check(text == "".join(chr(c) for c in range(ord(first), ord(last) + 1)), f"cp1140 {run}")
    for code, char in named.items():
        check(codecs.decode(bytes([code]), "cp1140") == char, f"cp1140 X'{code:02X}'")
    upper_span = range(0xC1, 0xE9 + 1)
    lower_span = range(0x81, 0xA9 + 1)
    upper_letters = sum(hi - lo + 1 for k, (lo, hi) in ebcdic.items() if k[0].isupper())
    lower_letters = sum(hi - lo + 1 for k, (lo, hi) in ebcdic.items() if k[0].islower())
    check((len(upper_span), upper_letters) == (41, 26), "A THRU Z spans 41 code points, 26 letters")
    check((len(lower_span), lower_letters) == (41, 26), "a THRU z spans 41 code points, 26 letters")
    check(0xD0 in upper_span and 0xE0 in upper_span, "} and backslash fall inside A THRU Z")
    check(0x9F in lower_span and 0xA1 in lower_span, "euro sign and tilde fall inside a THRU z")

    print("\nExpected standard output:")
    print("\n".join(console))
    print("\nExpected errors.txt:")
    print("\n".join(errors))
    print("\nExpected accepted.dat:")
    print("\n".join(goods))
    print("\nNUMVAL in 3.2 on the amounts TEST-NUMVAL refused (value, exception raised):")
    for k, (v, exc) in guesses.items():
        print(f"  {k:<10} -> {v}  {exc}")

    if len(sys.argv) == 3:
        for path, want in ((sys.argv[1], errors), (sys.argv[2], goods)):
            got = [ln.rstrip() for ln in open(path, encoding="ascii").read().splitlines()]
            diff = [(i + 1, g, w) for i, (g, w) in enumerate(zip(got, want)) if g != w]
            print(f"\n{path}: {len(got)} lines, expected {len(want)}, {len(diff)} differ")
            for i, g, w in diff:
                print(f"  line {i}\n    got      {g}\n    expected {w}")
            check(len(got) == len(want) and not diff, f"{path} matches")
    print(f"\nall {CHECKS} checks passed")


if __name__ == "__main__":
    main()