"""cobol-json-generate-parse.py - what cobol-json-generate-parse.cob should print and write, and the way back. The COBOL program turns a fixed-width account file into JSON Lines with JSON GENERATE. This companion works out, without a COBOL compiler, exactly what that program should produce, and then does the half GnuCOBOL 3.2 cannot do: it reads the JSON back into the fixed-width layout. Steps, in order: 1. Self-check. The rules below copy GnuCOBOL 3.2's JSON GENERATE formatting (libcob/mlio.c: get_num, get_trimmed_data, generate_json_from_tree; cJSON's unformatted printer). They are checked first against the expected results written into GnuCOBOL 3.2's own test suite (tests/testsuite.src/run_ml.at) and against the trimming examples in IBM's Enterprise COBOL for z/OS 6.4 Language Reference (SC27-8713-03, page 382). 2. Layout. Reads code/cobol-json-generate-parse.cpy, works out every field's width from its PICTURE, and checks the record is 91 bytes. Reads the NAME OF phrase, the receiving field's size and the console labels out of code/cobol-json-generate-parse.cob, so the expected output cannot drift from the listing. 3. Expected output. Reads datasets/cobol-json-generate-parse.dat and prints the console output of the COBOL program, then every line it should write to accounts.jsonl. 4. The way back. Parses each JSON line with Python's json module (numbers as Decimal), rebuilds the 91-byte record, and asserts that bytes 1-79 come back exactly. Bytes 80-91 (FILLER and the tax reference) are not in the JSON, by design. Optional: python code/cobol-json-generate-parse.py accounts.jsonl compares a file the COBOL program really wrote with the expected lines, then runs the way back on that file. Standard library only; Python 3.8 or later. Run from the site folder. """ from decimal import Decimal import json import re import sys CPY = "code/cobol-json-generate-parse.cpy" COB = "code/cobol-json-generate-parse.cob" DAT = "datasets/cobol-json-generate-parse.dat" CHECKS = 0 def check(cond, msg): global CHECKS assert cond, msg CHECKS += 1 # ------------------------------------------------- GnuCOBOL 3.2 JSON GENERATE rules --- def gc_num(value, int_digits, dec_digits): """libcob/mlio.c get_num(): MOVE to a numeric-edited item whose picture is '-' x max(int_digits, 1), then '9', then '.' and dec_digits 9s, then trim the leading spaces. Leading zeros go, one digit stays before the point, trailing decimal zeros stay.""" value = Decimal(value) text = f"{abs(value):.{dec_digits}f}" int_part = text.split(".")[0] assert len(int_part.lstrip("0") or "0") <= max(int_digits, 1), f"{value} does not fit" return ("-" if value < 0 else "") + text def gc_str(raw, justified=False): """libcob/mlio.c get_trimmed_data(): trailing spaces go (leading ones for JUSTIFIED RIGHT); a value that is all spaces keeps exactly one space.""" if justified: out = raw.lstrip(" ") else: out = raw.rstrip(" \x00") return out if out else " " def cjson_escape(s): """cJSON's string printer: quote and backslash escaped, control characters as \\b \\f \\n \\r \\t or \\u00XX, everything else (including '/') written as it is.""" out = [] for ch in s: if ch == '"': out.append('\\"') elif ch == "\\": out.append("\\\\") elif ch in "\b\f\n\r\t": out.append({"\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", "\t": "\\t"}[ch]) elif ord(ch) < 32: out.append(f"\\u{ord(ch):04x}") else: out.append(ch) return "".join(out) class Str: def __init__(self, name, raw, justified=False, suppressed=False): self.name, self.raw, self.justified, self.suppressed = name, raw, justified, suppressed class Num: def __init__(self, name, value, int_digits, dec_digits, suppressed=False): self.name, self.value, self.suppressed = name, value, suppressed self.int_digits, self.dec_digits = int_digits, dec_digits class Group: def __init__(self, name, children, suppressed=False): self.name, self.children, self.suppressed = name, children, suppressed def member(node): """One name/value pair, or '' for a group left with no children (mlio.c emits nothing).""" key = '"' + cjson_escape(node.name) + '":' if isinstance(node, Group): kids = [c for c in node.children if not c.suppressed] if not kids: return "" return key + "{" + ",".join(m for m in (member(c) for c in kids) if m) + "}" if isinstance(node, Num): return key + gc_num(node.value, node.int_digits, node.dec_digits) return key + '"' + cjson_escape(gc_str(node.raw, node.justified)) + '"' def json_generate(node, omit_name=False): """The text JSON GENERATE puts in the receiver: compact, no spaces, members in declaration order. omit_name is NAME OF IS OMITTED: the members go straight into the outer object.""" if omit_name: kids = [c for c in node.children if not c.suppressed] return "{" + ",".join(m for m in (member(c) for c in kids) if m) + "}" return "{" + member(node) + "}" def self_check(): # run_ml.at, "JSON GENERATE general" rec = lambda d=" ": Group("rec", [Str("alpha", "A "), Str("b", "BBB"), Group("c", [Str("ABCDEF", d)], suppressed=True)]) check(json_generate(rec()) == '{"rec":{"alpha":"A","b":"BBB"}}', "general test 1") check(json_generate(rec(), omit_name=True) == '{"alpha":"A","b":"BBB"}', "general test 2") check(json_generate(Str("d", " ")) == '{"d":" "}', "general test 3") check(json_generate(Group("c", [Str("d", " ")])) == '{"c":{"d":" "}}', "general test 4") check(json_generate(Str("d", '"""')) == '{"d":"\\"\\"\\""}', "general test 5") # "JSON GENERATE SUPPRESS" check(json_generate(Group("a", [Str("b", " ", suppressed=True), Str("c", " ")])) == '{"a":{"c":" "}}', "suppress test 1") check(json_generate(Group("x", [Group("y", [Str("z", " ", suppressed=True)])])) == '{"x":{}}', "suppress test 2") # "JSON GENERATE exceptions": the full text is 33 characters; a 5-byte receiver keeps the first 5 valid = json_generate(Group("valid-rec", [Str("a", "aa"), Str("b", "bb")])) check(len(valid) == 33 and valid[:5] == '{"val', "exceptions test: COUNT IN 33, receiver '{\"val'") # "JSON GENERATE record selection": the REDEFINES item d generated directly check(json_generate(Group("d", [Str("e", "abc000")])) == '{"d":{"e":"abc000"}}', "record selection") # "JSON GENERATE trimming" check(json_generate(Str("str", "ab ")) == '{"str":"ab"}', "trimming 1") check(json_generate(Str("num-edited", "01.00")) == '{"num-edited":"01.00"}', "trimming 2") check(json_generate(Num("leading-zeroes", 5, 5, 0)) == '{"leading-zeroes":5}', "trimming 3") check(json_generate(Group("decimal-nums", [Num("decimal-num", "0.12", 2, 3), Num("decimal-num-2", "0.1", 2, 1)])) == '{"decimal-nums":{"decimal-num":0.120,"decimal-num-2":0.1}}', "trimming 4") check(json_generate(Num("signed-decimal-num", -1, 3, 0)) == '{"signed-decimal-num":-1}', "trimming 5") check(json_generate(Num("comp-5-item", 5, 10, 0)) == '{"comp-5-item":5}', "trimming 6") check(json_generate(Num("index-item", 500, 9, 0)) == '{"index-item":500}', "trimming 7") check(json_generate(Str("just-item", " blah ", justified=True)) == '{"just-item":"blah "}', "trimming 10") check(json_generate(Num("integer-with-p", 10000, 6, 0)) == '{"integer-with-p":10000}', "trimming 11") check(json_generate(Num("decimal-with-p", "0.0004", 0, 4)) == '{"decimal-with-p":0.0004}', "trimming 12") # "JSON dpc-in-data config option" with -fdpc-in-data=none: PIC 9V9 VALUE 1,1 check(json_generate(Num("num", "1.1", 1, 1)) == '{"num":1.1}', "dpc-in-data none") # IBM 6.4 Language Reference p. 382, "Trimming of generated JSON data" check(gc_num("-12.340", 3, 3) == "-12.340", "IBM: -012.340 becomes -12.340") check(gc_num("0.45", 4, 2) == "0.45", "IBM: 0000.45 becomes 0.45") check(gc_num("13", 4, 0) == "13", "IBM: 0013 becomes 13") check(gc_num("0", 4, 0) == "0", "IBM: 0000 becomes 0") # ---------------------------------------------------------------- layout checks --- def pic_size(pic, separate): s = re.sub(r"(.)\((\d+)\)", lambda m: m.group(1) * int(m.group(2)), pic.upper()) size = len(s.replace("S", "").replace("V", "")) return size + (1 if "S" in s and separate else 0) def read_copybook(): text = open(CPY, encoding="ascii").read() code = "\n".join(line.split("*>")[0] for line in text.splitlines()) entries = re.findall(r"(\d\d)\s+([A-Za-z0-9-]+)\s*([^.]*)\.", code) fields = [] for level, name, rest in entries: if level in ("01", "88"): continue pic = re.search(r"PIC\s+(\S+)", rest) redefines = "REDEFINES" in rest fields.append({"level": level, "name": name, "redefines": redefines, "size": pic_size(pic.group(1), "SEPARATE" in rest) if pic else None, "pic": pic.group(1) if pic else None}) return fields def read_listing(): text = open(COB, encoding="ascii").read() block = re.search(r"NAME OF Json-Account IS OMITTED,(.*?)ON EXCEPTION", text, flags=re.S).group(1) names = re.findall(r"(J-[A-Za-z]+)\s+IS\s+\"([^\"]+)\"", block) receiver = int(re.search(r"01\s+WS-Json\s+PIC X\((\d+)\)", text).group(1)) labels = re.findall(r'DISPLAY "([^"]+: )" WS-Show', text) return text, names, receiver, labels # ------------------------------------------------------------------- the data --- STATUS_WORDS = {"A": "active", "D": "dormant", "C": "closed"} def load_records(): recs = [] for n, line in enumerate(open(DAT, encoding="ascii").read().split("\n"), 1): if not line: continue check(len(line) == 91, f"line {n}: {len(line)} bytes, expected 91") r = {"line": line, "id": line[0:8], "holder": line[8:32], "joint": line[32:56], "branch": line[56:60], "sign": line[60], "digits": line[61:70], "opened": line[70:78], "status": line[78], "filler": line[79:82], "tax": line[82:91]} # the program's NUMERIC tests: a + or - then nine digits passes for SIGN LEADING SEPARATE check(r["branch"].isdigit() and r["opened"].isdigit() and r["digits"].isdigit() and r["sign"] in "+-", f"line {n}: a numeric field is not numeric") r["balance"] = Decimal(r["digits"]) / 100 * (-1 if r["sign"] == "-" else 1) recs.append(r) return recs def raw_tree(r, fields): """Acct-Rec as JSON GENERATE sees it: FILLER and the REDEFINES skipped, the tax ref suppressed.""" values = {"Acct-Id": Str("Acct-Id", r["id"]), "Acct-Holder": Str("Acct-Holder", r["holder"]), "Acct-Joint": Str("Acct-Joint", r["joint"]), "Acct-Branch": Num("Acct-Branch", r["branch"], 4, 0), "Acct-Balance": Num("Acct-Balance", r["balance"], 7, 2), "Acct-Opened": Num("Acct-Opened", r["opened"], 8, 0), "Acct-Status": Str("Acct-Status", r["status"]), "Acct-Tax-Ref": Str("Acct-Tax-Ref", r["tax"], suppressed=True)} kids = [] for f in fields: if f["level"] != "05" or f["redefines"] or f["name"].upper() == "FILLER": continue kids.append(values[f["name"]]) return Group("Acct-Rec", kids) def view_tree(r, names): n = dict(names) o = r["opened"] return Group("Json-Account", [ Str(n["J-Id"], r["id"]), Str(n["J-Holder"], r["holder"]), Str(n["J-Joint"], r["joint"]), Str(n["J-Branch"], r["branch"]), Num(n["J-Balance"], r["balance"], 7, 2), Str(n["J-Opened"], f"{o[0:4]}-{o[4:6]}-{o[6:8]}"), Str(n["J-Status"], STATUS_WORDS.get(r["status"], "unknown").ljust(7))]) def spaces_outside_strings(s): """Count the spaces that are not inside a JSON string (cJSON's unformatted printer adds none).""" inside, escaped, count = False, False, 0 for ch in s: if inside: if escaped: escaped = False elif ch == "\\": escaped = True elif ch == '"': inside = False elif ch == '"': inside = True elif ch == " ": count += 1 return count def z5(n): # PIC ZZZZ9 return f"{n}".rjust(5) def money(d): # PIC -,---,---,--9.99 (floating minus), 16 characters s = f"{abs(d):,.2f}" return (("-" + s) if d < 0 else s).rjust(16) def way_back(line, names): """JSON text -> the first 79 bytes of the record, the way a reader of the JSON would do it.""" rev = {v: k for k, v in names} obj = json.loads(line, parse_float=Decimal) check(list(obj) == [v for _, v in names], f"member order {list(obj)}") f = {rev[k]: v for k, v in obj.items()} cents = int(f["J-Balance"] * 100) letter = {w: c for c, w in STATUS_WORDS.items()}[f["J-Status"]] return (f["J-Id"].ljust(8) + f["J-Holder"].ljust(24) + f["J-Joint"].ljust(24) + f["J-Branch"] + ("-" if cents < 0 else "+") + f"{abs(cents):09d}" + f["J-Opened"].replace("-", "") + letter) def main(): sys.stdout.reconfigure(encoding="utf-8") self_check() fields = read_copybook() top = [f for f in fields if f["level"] == "05" and not f["redefines"]] check(sum(f["size"] for f in top) == 91, "copybook: the 05 levels must add up to 91 bytes") parts = [f for f in fields if f["level"] == "10"] check(sum(f["size"] for f in parts) == 8, "copybook: the REDEFINES parts must cover 8 bytes") check([f["size"] for f in top] == [8, 24, 24, 4, 10, 8, 1, 3, 9], "copybook: field widths") text, names, receiver, labels = read_listing() check(names == [("J-Id", "accountId"), ("J-Holder", "holder"), ("J-Joint", "jointHolder"), ("J-Branch", "branch"), ("J-Balance", "balance"), ("J-Opened", "opened"), ("J-Status", "status")], f"NAME OF phrase in the listing: {names}") check(receiver == 300, "receiving field PIC X(300)") check(labels == ["Records read : ", "JSON lines written: ", "Records failed : ", "Longest line : ", "Balance total : "], f"console labels {labels}") code = "\n".join(line.split("*>")[0] for line in text.splitlines()) check(len(re.findall(r"END-DISPLAY\s+NOT ON EXCEPTION", code)) == 2, "END-DISPLAY closes the DISPLAY in both ON EXCEPTION branches") check(len(re.findall(r"^\s+JSON GENERATE WS-Json FROM ", code, flags=re.M)) == 2 and "JSON PARSE" not in code, "two JSON GENERATE statements and no JSON PARSE in the code") recs = load_records() check(len(recs) == 8, "eight records") raw = json_generate(raw_tree(recs[0], fields)) lines = [json_generate(view_tree(r, names), omit_name=True) for r in recs] total = sum(r["balance"] for r in recs) longest = max(len(x) for x in lines) check(all(len(x) <= receiver for x in lines), "every line fits the receiving field") check(len(raw) <= receiver, "the raw JSON fits too") view_size = 8 + 24 + 24 + 4 + 9 + 10 + 7 check(view_size == 86, "Json-Account is 86 bytes") console = ["Straight from the copybook:", raw, "Records read : " + z5(len(recs)), "JSON lines written: " + z5(len(lines)), "Records failed : " + z5(0), "Longest line : " + z5(longest), "Balance total : " + money(total)] print("--- standard output of the COBOL program ---") print("\n".join(console)) print("--- accounts.jsonl ---") print("\n".join(lines)) # the console line for record 1, field by field check(raw == '{"Acct-Rec":{"Acct-Id":"10004217","Acct-Holder":"Ada Lovell","Acct-Joint":" ",' '"Acct-Branch":42,"Acct-Balance":1234.50,"Acct-Opened":20190314,"Acct-Status":"A"}}', "raw JSON for record 1") check("Acct-Tax-Ref" not in raw and "TR0004217" not in raw, "SUPPRESS keeps the tax reference out") check("Acct-Opened-Year" not in raw and "FILLER" not in raw, "REDEFINES and FILLER are skipped") check(json.loads(raw)["Acct-Rec"]["Acct-Branch"] == 42, "the branch 0042 arrives as the number 42") # the JSON Lines file, line by line check(lines[0] == '{"accountId":"10004217","holder":"Ada Lovell","jointHolder":" ","branch":"0042",' '"balance":1234.50,"opened":"2019-03-14","status":"active"}', "line 1") check('"holder":"Mae \\"Dot\\" Fenwick"' in lines[1], "line 2: the quotes are escaped") check('"balance":-75.25' in lines[2], "line 3: a negative balance") check('"balance":0.05' in lines[3] and '"status":"dormant"' in lines[3], "line 4: 0.05, dormant") check('"balance":0.00' in lines[5] and '"status":"closed"' in lines[5], "line 6: zero keeps 0.00") check('"holder":"Margaret Ashworth-Barnes"' in lines[7], "line 8: a full 24-character name") check(sum(x.count('"jointHolder":" "') for x in lines) == 6, "six accounts have no joint holder") check(all("TR000" not in x for x in lines), "no tax reference in any line") check(not any(spaces_outside_strings(x) for x in lines + [raw]), "compact: no space outside a string") check(total == Decimal("74490.30"), f"balance total {total}") check([len(x) for x in lines] == [138, 159, 141, 149, 157, 138, 145, 152], "line lengths") check(longest == len(lines[1]) == 159, f"longest line {longest}: the record with quotes and a joint holder") check(len(raw) == 160, f"raw JSON for record 1 is {len(raw)} characters") check((Decimal(longest) / view_size).quantize(Decimal("0.01")) == Decimal("1.85"), "longest line / Json-Account = 1.85, under IBM's 2 to 3") # the way back parsed_total = Decimal(0) for r, x in zip(recs, lines): back = way_back(x, names) check(back == r["line"][:79], f"round trip of {r['id']}: {back!r}") parsed_total += json.loads(x, parse_float=Decimal)["balance"] check(parsed_total == total, "the JSON carries the same money as the file") if len(sys.argv) > 1: real = [x for x in open(sys.argv[1], encoding="utf-8").read().split("\n") if x] for n, (got, want) in enumerate(zip(real, lines), 1): if got != want: print(f"line {n} differs:\n got {got}\n want {want}") print(f"{sys.argv[1]}: {len(real)} lines, {sum(g == w for g, w in zip(real, lines))} as expected") for r, x in zip(recs, real): if way_back(x, names) != r["line"][:79]: print(f"{r['id']}: does not come back byte for byte") print(f"--- {CHECKS} checks passed; longest line {longest} of {receiver}; " f"Json-Account {view_size} bytes; total {total} ---") if __name__ == "__main__": main()