"""cobol-inspect-replacing-converting.py - the expected output of cobol-inspect.cob What it does: copies the INSPECT routines of the GnuCOBOL 3.2 runtime (libcob/strings.c: cob_inspect_init, _start, _before, _after, _characters, _all, _leading, _first, _trailing, _converting and _finish) and the compile-time size check from cobc/typeck.c (validate_inspect), checks those copies against the INSPECT cases in the GnuCOBOL 3.2 test suite and against the worked examples in IBM's Enterprise COBOL 6.4 Language Reference and the GnuCOBOL Programmer's Guide, and then walks the order file the way the COBOL program does, printing what the program should display. Why it exists: the COBOL listing was desk-checked, not compiled. This file is how the output shown in the lesson was worked out. When the program is compiled and run, its output can be compared with this one line by line. Inputs : datasets/cobol-inspect-replacing-converting.dat (or a path given as argument 1) Run : python code/cobol-inspect-replacing-converting.py (from the site folder) python code/cobol-inspect-replacing-converting.py --selftest (the checks only) python code/cobol-inspect-replacing-converting.py --compare (one phrase order case) Needs : Python 3.8 or later, standard library only """ import sys from pathlib import Path RECORD = 100 # code/cobol-inspect-replacing-converting.cpy LAYOUT = [("Ord-Id", 6), ("Ord-Name", 20), ("Ord-Date", 10), ("Ord-Qty", 4), ("Ord-Email", 36), ("Ord-Note", 24)] class Fig(bytes): """A figurative constant (SPACE, ZERO ...): one character, repeated as needed (alloc_figurative in strings.c). typeck.c sizes it as 1.""" SPACE, ZERO, LOW = Fig(b" "), Fig(b"0"), Fig(b"\x00") class Counter: """A TALLYING counter; cob_add_int adds to it, nothing clears it.""" def __init__(self, v=0): self.v = v # ---------------------------------------------------------------------------- # Port of the INSPECT part of libcob/strings.c (GnuCOBOL 3.2). The subject is a # bytearray (or a slice view described by base + offset, for reference # modification). Positions are 0-based like the C pointers. # ---------------------------------------------------------------------------- class Inspect: def __init__(self, data, offset=0, length=None, replacing=False): # cob_inspect_init: the subject, a cleared marker, REPLACING flag self.base = data self.off = offset self.size = len(data) - offset if length is None else length self.replacing = replacing self.mark = [0] * self.size self.repdata = [None] * self.size self.start = self.end = None def data(self): return bytes(self.base[self.off:self.off + self.size]) # cob_inspect_start / _before / _after def region_start(self): self.start, self.end = 0, self.size def _find(self, s): # inspect_find_data: search from the current start to the current end d = self.data() s = bytes(s) p, end_p = self.start, self.end - len(s) + 1 while p < end_p: if d[p:p + len(s)] == s: return p p += 1 return None def before(self, s): p = self._find(s) if p is not None: self.end = p def after(self, s): p = self._find(s) if p is not None: self.start = p + len(s) else: self.start = self.end def region(self, regs): """Run the region calls in the order written (parser.y inspect_region).""" self.region_start() for kind, s in regs: (self.before if kind == "BEFORE" else self.after)(s) def _is_marked(self, pos, n): return any(self.mark[pos:pos + n]) def _set_mark(self, pos, n): for i in range(pos, pos + n): self.mark[i] = 1 def characters(self, f1): n = self.end - self.start if n <= 0: return if self.replacing: for i in range(self.start, self.end): if not self.mark[i]: self.repdata[i] = bytes(f1)[0] else: free = sum(1 for i in range(self.start, self.end) if not self.mark[i]) if free: f1.v += free self._set_mark(self.start, n) def common(self, f1, f2, kind): """inspect_common: f1 = counter (TALLYING) or replacement, f2 = pattern.""" inspect_len = self.end - self.start if inspect_len == 0: return f2 = bytes(f2) if not self.replacing: if len(f2) > inspect_len: return self._no_replace(f1, f2, kind) else: if len(f1) != len(f2): if isinstance(f1, Fig): f1 = bytes(f1) * len(f2) # alloc_figurative else: raise RuntimeError("EC-RANGE-INSPECT-SIZE") if len(f2) > inspect_len: return self._replace(bytes(f1), f2, kind) def _no_replace(self, counter, f2, kind): d, s, L, n = self.data(), self.start, self.end - self.start, 0 if kind == "TRAILING": i, first = L - len(f2), 0 while True: if d[s + i:s + i + len(f2)] == f2: if not self._is_marked(s + i, len(f2)): n += 1 first = i i -= len(f2) - 1 if i == 0: break i -= 1 else: break if n: self._set_mark(s + first, L - first) elif kind == "LEADING": i, last = 0, 0 while i < L - len(f2) + 1: if d[s + i:s + i + len(f2)] == f2: if not self._is_marked(s + i, len(f2)): n += 1 i += len(f2) - 1 last = i else: break i += 1 if n: self._set_mark(s, last) else: # ALL, FIRST i = 0 while i < L - len(f2) + 1: if d[s + i:s + i + len(f2)] == f2: if not self._is_marked(s + i, len(f2)): n += 1 self._set_mark(s + i, len(f2)) if kind == "FIRST": break i += len(f2) - 1 i += 1 if n: counter.v += n def _do_mark(self, pos, rep): if self._is_marked(pos, len(rep)): return False for k, b in enumerate(rep): self.repdata[pos + k] = b self._set_mark(pos, len(rep)) return True def _replace(self, rep, f2, kind): d, s, L = self.data(), self.start, self.end - self.start if kind == "TRAILING": i = L - len(f2) while True: if d[s + i:s + i + len(f2)] == f2: if self._do_mark(s + i, rep): i -= len(f2) - 1 if i == 0: break i -= 1 else: break elif kind == "LEADING": i = 0 while i < L - len(f2) + 1: if d[s + i:s + i + len(f2)] == f2: if self._do_mark(s + i, rep): i += len(f2) - 1 else: break i += 1 else: # ALL, FIRST i = 0 while i < L - len(f2) + 1: if d[s + i:s + i + len(f2)] == f2: if self._do_mark(s + i, rep): if kind == "FIRST": break i += len(f2) - 1 i += 1 def finish(self): """cob_inspect_finish: copy the replacements back, all at once.""" if self.replacing: for i, b in enumerate(self.repdata): if self.mark[i] and b is not None: self.base[self.off + i] = b def converting(self, frm, to): """cob_inspect_converting: one table, the first occurrence of a duplicate "from" character wins; changes the data directly.""" n = self.end - self.start if n == 0: return frm = bytes(frm) if len(frm) != len(to): if isinstance(to, Fig): to = bytes(to) * len(frm) else: raise RuntimeError("EC-RANGE-INSPECT-SIZE") to = bytes(to) table = {} for a, b in zip(frm, to): table.setdefault(a, b) for i in range(self.start, self.end): c = self.base[self.off + i] if c in table: self.base[self.off + i] = table[c] # Statement-level helpers: one call per INSPECT statement, the way codegen # emits it (cb_emit_inspect: init, one region + routine per phrase, finish). def tallying(subject, phrases, offset=0, length=None): """phrases: (counter, kind, pattern_or_None, regions).""" ins = Inspect(subject, offset, length, replacing=False) for counter, kind, pat, regs in phrases: ins.region(regs) if kind == "CHARACTERS": ins.characters(counter) else: ins.common(counter, pat, kind) ins.finish() def replacing(subject, phrases, offset=0, length=None): """phrases: (kind, pattern_or_None, replacement, regions).""" ins = Inspect(subject, offset, length, replacing=True) for kind, pat, rep, regs in phrases: ins.region(regs) if kind == "CHARACTERS": ins.characters(rep) else: ins.common(rep, pat, kind) ins.finish() def converting(subject, frm, to, regs=(), offset=0, length=None): ins = Inspect(subject, offset, length) ins.region(regs) ins.converting(frm, to) def validate_inspect(x, y): """cobc/typeck.c validate_inspect: an error when both operands have a known size and the sizes differ. A figurative constant (CB_CONST) counts as size 1 on the left and as 'no size' on the right. Returns True if OK.""" size1 = 1 if isinstance(x, Fig) else len(x) size2 = 0 if isinstance(y, Fig) else len(y) return not (size1 and size2 and size1 != size2) def ibm_replacing(subject, phrases): """IBM's comparison cycle (LR 6.4 pp. 360-361) for REPLACING phrases without BEFORE/AFTER: at each position try the comparands in the order written; on a match replace and move past it, otherwise move one place. Comparisons read the original value. Used only by --compare.""" src, out, i = bytes(subject), bytearray(subject), 0 while i < len(src): for kind, pat, rep in phrases: pat = bytes(pat) if src[i:i + len(pat)] == pat: out[i:i + len(pat)] = rep i += len(pat) break else: i += 1 return bytes(out) # ---------------------------------------------------------------------------- # Self-test: the INSPECT cases of the GnuCOBOL 3.2 test suite, IBM's worked # examples (LR 6.4 pp. 356-360) and the Programmer's Guide's examples (7.8.24). # ---------------------------------------------------------------------------- def selftest(): results = [] def ok(name, cond): results.append((name, bool(cond))) def B(s): return bytearray(s, "ascii") # run_misc.at: INSPECT REPLACING LEADING ZEROS BY SPACES x = B("0001") replacing(x, [("LEADING", ZERO, SPACE, [])]) ok("suite: REPLACING LEADING ZEROS BY SPACES -> ' 1'", x == b" 1") # run_misc.at: INSPECT: no repeat conversion x, y = B("BCA"), B(" BCA") converting(x, b"ABC", b"BCD") converting(y, b"ABC", b"BCD") ok("suite: no repeat conversion X -> CDB", x == b"CDB") ok("suite: no repeat conversion Y -> ' CDB'", y == b" CDB") # run_misc.at: INSPECT CONVERTING TO figurative constant x = B("BCA") converting(x, b"ABC", SPACE) ok("suite: CONVERTING 'ABC' TO SPACES -> spaces", x == b" ") # run_misc.at: INSPECT CONVERTING complex x = B("AZABBCDCCECC") converting(x, b"ABZC", b"ZY0X") ok("suite: CONVERTING 'ABZC' TO 'ZY0X'", x == b"Z0ZYYXDXXEXX") converting(x, b"XD", SPACE, [("BEFORE", b"E"), ("AFTER", b"D")]) ok("suite: CONVERTING 'XD' TO SPACES BEFORE 'E' AFTER 'D'", x == b"Z0ZYYXD EXX") # run_misc.at: INSPECT TALLYING BEFORE / AFTER / BEFORE and AFTER for val, regs, want, name in [ ("ABC ", [("BEFORE", b" ")], 3, "TALLYING CHARACTERS BEFORE ' ' on 'ABC '"), (" ABC", [("BEFORE", b" ")], 0, "TALLYING CHARACTERS BEFORE ' ' on ' ABC'"), ("ABC ", [("AFTER", b" ")], 0, "TALLYING CHARACTERS AFTER ' ' on 'ABC '"), (" ABC", [("AFTER", b" ")], 3, "TALLYING CHARACTERS AFTER ' ' on ' ABC'"), ("ABC ", [("BEFORE", b" "), ("AFTER", b" ")], 0, "TALLYING CHARACTERS BEFORE ' ' AFTER ' '"), (" " * 256, [("AFTER", b"<"), ("BEFORE", b" ' A'", x == b" A") # syn_misc.at: INSPECT invalid size (compile-time errors on lines 14, 15, # 17, 18, 23; lines 16, 19 and 22 compile) ok("suite: line 14 REPLACING ALL SPACES BY 'AA' is an error", not validate_inspect(SPACE, b"AA")) ok("suite: line 15 REPLACING ALL 'ABC' BY 'AA' is an error", not validate_inspect(b"ABC", b"AA")) ok("suite: line 16 REPLACING ALL 'DEF' BY SPACES compiles", validate_inspect(b"DEF", SPACE)) ok("suite: line 17 CONVERTING SPACES TO 'AA' is an error", not validate_inspect(SPACE, b"AA")) ok("suite: line 18 CONVERTING 'ABC' TO 'AA' is an error", not validate_inspect(b"ABC", b"AA")) ok("suite: line 19 CONVERTING 'DEF' TO SPACES compiles", validate_inspect(b"DEF", SPACE)) ok("suite: line 22 CONVERTING 'DEF' TO X(2)(X-POS:3) compiles", validate_inspect(b"DEF", b"x" * 3)) ok("suite: line 23 CONVERTING 'DEF' TO X(2)(X-POS:4) is an error", not validate_inspect(b"DEF", b"x" * 4)) # IBM LR 6.4 p. 356: TALLYING phrase example c = Counter() tallying(B("In order to form a"), [(c, "ALL", SPACE, [])]) ok("IBM: 'In order to form a' has 4 spaces", c.v == 4) # IBM LR 6.4 p. 357: REPLACING phrase example (format 3) s, c = B("more,perfect,union"), Counter() tallying(s, [(c, "ALL", b",", [])]) replacing(s, [("ALL", b",", SPACE, [])]) ok("IBM: commas replaced = 2", c.v == 2) ok("IBM: CHARS is now 'more perfect union'", s == b"more perfect union") # IBM LR 6.4 p. 358: BEFORE and AFTER phrases example s = B("$some.confusing_text with.hyphens-periods.spaces".ljust(50)) c1, c2, c3 = Counter(), Counter(), Counter() tallying(s, [(c1, "CHARACTERS", None, [("AFTER", b"$"), ("BEFORE", b"_")]), (c2, "ALL", b".", [("AFTER", b"h")]), (c3, "ALL", b".", [("BEFORE", b"a")])]) replacing(s, [("ALL", b".", SPACE, [("BEFORE", b"h")]), ("ALL", b'"', SPACE, [("AFTER", b"s")]), ("ALL", b"_", SPACE, [])]) ok("IBM: characters after $ and before _ = 14", c1.v == 14) ok("IBM: periods after h = 2", c2.v == 2) ok("IBM: periods before initial a = 0", c3.v == 0) ok("IBM: CHARS is now '$some confusing text with.hyphens-periods.spaces'", bytes(s).rstrip() == b"$some confusing text with.hyphens-periods.spaces") # IBM LR 6.4 p. 360: CONVERTING phrase example, Table 40 for before_, after_ in [("a/five/?six", "a/FIVE/?six"), ("r/Rexx/RRRr", "r/REXX/RRRR"), ("zfour?inspe", "zfour?inspe")]: s = B(before_) converting(s, b"abcdefghijklmnopqrstuvwxyz", b"ABCDEFGHIJKLMNOPQRSTUVWXYZ", [("AFTER", b"/"), ("BEFORE", b"?")]) ok(f"IBM: Table 40 {before_} -> {after_}", s == after_.encode()) # Programmer's Guide 7.8.24 examples c1, c2 = Counter(), Counter() tallying(B("--ABCDEF----BCDEF--"), [(c1, "ALL", b"ABC", []), (c2, "ALL", b"BCDE", [])]) ok("guide: C-ABC = 1 and C-BCDE = 1", c1.v == 1 and c2.v == 1) s = B("--ABCDEF----BCDEF--") replacing(s, [("ALL", b"ABC", b"DEF", []), ("ALL", b"BCDE", b"WXYZ", [])]) ok("guide: --DEFDEF----WXYZF--", s == b"--DEFDEF----WXYZF--") for kind, pat, want in [("ALL", b"XX", 3), ("LEADING", b"XX", 0), ("LEADING", b"a", 1), ("TRAILING", b"d", 2)]: c = Counter() tallying(B("aXXabbXXccXXdd"), [(c, kind, pat, [])]) ok(f"guide: {kind} {pat.decode()!r} in aXXabbXXccXXdd = {want}", c.v == want) s = B("aXXabbXXccXXdd") replacing(s, [("FIRST", b"XX", b"YY", [])]) ok("guide: FIRST 'XX' is the one between a and b", s == b"aYYabbXXccXXdd") for name, passed in results: print(("ok " if passed else "FAIL ") + name) good = sum(p for _, p in results) print(f"{good} of {len(results)} checks passed") return good == len(results) def compare(): """One case where the order of the phrases matters: GnuCOBOL 3.2 runs each phrase over the whole range in turn; IBM's comparison cycle tries every phrase at one position before moving on.""" phrases = [("ALL", b"B", b"x"), ("ALL", b"AB", b"yy")] s = bytearray(b"AB") replacing(s, [(k, p, r, []) for k, p, r in phrases]) print('INSPECT S REPLACING ALL "B" BY "x" ALL "AB" BY "yy", S = "AB"') print("GnuCOBOL 3.2 runtime (as read): " + bytes(s).decode()) print("IBM comparison cycle: " + ibm_replacing(b"AB", phrases).decode()) # ---------------------------------------------------------------------------- # The program, paragraph by paragraph. # ---------------------------------------------------------------------------- LOWER = b"abcdefghijklmnopqrstuvwxyz" UPPER = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" KEY_FROM = b"abcdefghijklmnopqrstuvwxyz'.-," KEY_TO = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ " def is_numeric(b): """NUMERIC on PIC X: every character a digit.""" return len(b) > 0 and all(48 <= c <= 57 for c in b) def read_records(path): """LINE SEQUENTIAL: one record per line, CR before LF dropped, short lines padded with spaces to the record size (lineseq_read, libcob/fileio.c).""" for raw in Path(path).read_bytes().split(b"\n"): if raw.endswith(b"\r"): raw = raw[:-1] yield raw def run(path): lines = list(read_records(path)) if lines and lines[-1] == b"": lines.pop() out, n_read, n_clean, n_rej = [], 0, 0, 0 for raw in lines: assert len(raw) <= RECORD rec = bytearray(raw.ljust(RECORD, b" ")) f, pos = {}, 0 for name, w in LAYOUT: f[name] = (pos, w) pos += w def get(name): p, w = f[name] return bytes(rec[p:p + w]) n_read += 1 reason, bad, bad_len = b"", b"", 0 # Clean-Qty p, w = f["Ord-Qty"] bad, bad_len = get("Ord-Qty"), 4 if get("Ord-Qty") == b" " * 4: reason = b"QUANTITY IS BLANK" else: replacing(rec, [("LEADING", SPACE, ZERO, [])], p, w) if not is_numeric(get("Ord-Qty")): bad = get("Ord-Qty") reason = b"QUANTITY IS NOT A NUMBER" # Clean-Date if not reason: p, w = f["Ord-Date"] converting(rec, b"/.", b"--", (), p, w) d = get("Ord-Date") if (not is_numeric(d[0:4]) or d[4:5] != b"-" or not is_numeric(d[5:7]) or d[7:8] != b"-" or not is_numeric(d[8:10])): bad, bad_len = d, 10 reason = b"DATE IS NOT YYYY-MM-DD" # Clean-Email mask = None if not reason: p, w = f["Ord-Email"] bad = get("Ord-Email") c = Counter(0) tallying(rec, [(c, "CHARACTERS", None, [("BEFORE", SPACE)])], p, w) bad_len = c.v if c.v else 1 ats = Counter(0) tallying(rec, [(ats, "ALL", b"@", [])], p, w) if ats.v == 0: reason = b"E-MAIL HAS NO @" elif ats.v > 1: reason = b"E-MAIL HAS MORE THAN ONE @" elif get("Ord-Email")[0:1] == b"@": reason = b"E-MAIL HAS NOTHING BEFORE THE @" else: converting(rec, UPPER, LOWER, [("AFTER", b"@")], p, w) mask = bytearray(get("Ord-Email")) replacing(mask, [("CHARACTERS", None, b"*", [("BEFORE", b"@")])], 1) n = "{:>3}".format(n_read) oid = get("Ord-Id").decode() if not reason: # Clean-Note p, w = f["Ord-Note"] bars = Counter(0) tallying(rec, [(bars, "ALL", b"|", [])], p, w) replacing(rec, [("ALL", b"|", b"/", [])], p, w) converting(rec, LOWER, UPPER, [("AFTER", b"["), ("BEFORE", b"]")], p, w) # Build-Key key = bytearray(get("Ord-Name")) converting(key, KEY_FROM, KEY_TO) # Show-Clean out.append(f"REC {n} {oid} OK {key.decode()} {get('Ord-Date').decode()}" f" QTY {get('Ord-Qty').decode()}") out.append(" EMAIL " + get("Ord-Email").decode().rstrip(" ")) out.append(" MASKED " + bytes(mask).decode().rstrip(" ")) note = get("Ord-Note").decode() if note.strip(" "): if bars.v > 0: out.append(" NOTE " + note.rstrip(" ") + " (BARS REPLACED:" + "{:>2}".format(bars.v) + ")") else: out.append(" NOTE " + note.rstrip(" ")) n_clean += 1 else: out.append(f"REC {n} {oid} REJECT {reason.decode()} " f"[{bad[:bad_len].decode()}]") n_rej += 1 out.append("----") out.append("RECORDS READ {:>3}".format(n_read)) out.append("CLEANED {:>3}".format(n_clean)) out.append("REJECTED {:>3}".format(n_rej)) return out, (4 if n_rej else 0) def main(argv): if "--selftest" in argv: return 0 if selftest() else 1 if "--compare" in argv: compare() return 0 if not _quiet_selftest(): print("self-test failed; run with --selftest") return 1 here = Path(__file__).resolve().parent.parent path = argv[1] if len(argv) > 1 else here / "datasets" / "cobol-inspect-replacing-converting.dat" out, rc = run(path) print("\n".join(out)) print(f"(return code {rc})") return 0 def _quiet_selftest(): import io import contextlib with contextlib.redirect_stdout(io.StringIO()): return selftest() if __name__ == "__main__": sys.exit(main(sys.argv))