264 lines
13 KiB
Python
264 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Assertion-integrity analyzer for the E2E Council pipeline (deterministic — no LLM).
|
|
|
|
Two modes:
|
|
fingerprint <spec.js> -> print a JSON fingerprint of the spec's assertions
|
|
check <spec.js> [--baseline <fingerprint.json>] [--allow-weakening] [--strengthen]
|
|
-> static anti-pattern scan + (optional) diff-aware
|
|
heal-weakening check; writes nothing, prints a JSON
|
|
verdict, exits 2 if any CRITICAL finding.
|
|
--strengthen (Refiner mode): permit ADDED negative
|
|
matchers (assert-absence is valid strengthening);
|
|
still blocks any DROP of assertions.
|
|
|
|
Conservative by design: only HIGH-CONFIDENCE patterns are flagged as critical, because a gate with
|
|
false positives gets disabled by humans (the same trap that made mutation-testing a poor fit).
|
|
"""
|
|
import sys, re, json
|
|
|
|
NEG_MATCHERS = re.compile(r'\.not\.|\.toBeHidden\(|\.toBeFalsy\(')
|
|
TAUTOLOGIES = [
|
|
(re.compile(r'toBeGreaterThanOrEqual\(\s*0\s*\)'), 'count >= 0 is always true'),
|
|
(re.compile(r'toBeGreaterThan\(\s*-1\s*\)'), 'count > -1 is always true'),
|
|
(re.compile(r'toBeLessThan\(\s*Infinity\s*\)'), '< Infinity is always true'),
|
|
(re.compile(r'expect\(\s*true\s*\)\.toBe(Truthy)?\('), 'expect(true) asserts nothing'),
|
|
(re.compile(r'expect\(\s*1\s*\)\.toBe\(\s*1\s*\)'), 'expect(1).toBe(1) asserts nothing'),
|
|
(re.compile(r'expect\(\s*false\s*\)\.toBe(Falsy)?\('), 'expect(false) asserts nothing'),
|
|
]
|
|
EXPECT = re.compile(r'\bexpect\(')
|
|
TEST_START = re.compile(r'\b(test|it)(\.skip|\.fixme|\.only)?\s*\(')
|
|
SKIP_FIXME = re.compile(r'\b(test|it)\.(skip|fixme)\s*\(|\btest\.skip\(\s*\)|\.fixme\(')
|
|
|
|
|
|
def strip_noise(line):
|
|
"""Blank out comments + string literals so matchers don't fire inside them (best-effort,
|
|
line-local). Handles `//` line comments and single-line `/* */` blocks; multi-line `/* */`
|
|
blocks are rare in generated specs and not tracked across lines."""
|
|
line = re.sub(r'/\*.*?\*/', '', line) # single-line block comment
|
|
line = re.sub(r'//.*$', '', line) # line comment
|
|
line = re.sub(r'`[^`]*`', '``', line)
|
|
line = re.sub(r"'[^']*'", "''", line)
|
|
line = re.sub(r'"[^"]*"', '""', line)
|
|
return line
|
|
|
|
|
|
def fingerprint(src):
|
|
"""Return counts that summarize a spec's assertion shape: total expect()s, negative matchers,
|
|
skip/fixme, and test blocks. Compared before vs. after heal to detect weakening."""
|
|
code = '\n'.join(strip_noise(l) for l in src.split('\n'))
|
|
return {
|
|
'expects': len(EXPECT.findall(code)),
|
|
'negatives': len(NEG_MATCHERS.findall(code)),
|
|
'skips': len(re.findall(r'\.(skip|fixme)\s*\(', code)),
|
|
'tests': len(TEST_START.findall(code)),
|
|
}
|
|
|
|
|
|
def find_test_blocks(lines):
|
|
"""Yield (name_line_idx, start_idx, end_idx) for each test()/it() block by matching the block's
|
|
braces. Ends exactly when the brace depth returns to 0 (== 0, not <= 0, so a stray nested
|
|
structure can't terminate the block early)."""
|
|
blocks = []
|
|
i, n = 0, len(lines)
|
|
while i < n:
|
|
if TEST_START.search(strip_noise(lines[i])):
|
|
depth, j, started = 0, i, False
|
|
while j < n:
|
|
for ch in strip_noise(lines[j]):
|
|
if ch == '{':
|
|
depth += 1; started = True
|
|
elif ch == '}':
|
|
depth -= 1
|
|
if started and depth == 0:
|
|
break
|
|
j += 1
|
|
blocks.append((i, i, j))
|
|
i = j + 1
|
|
else:
|
|
i += 1
|
|
return blocks
|
|
|
|
|
|
# `if (...)` / `else` that directly precedes a statement on the SAME line with NO opening brace —
|
|
# i.e. a braceless single-statement conditional like `if (x) await expect(y).toBeVisible();`.
|
|
BRACELESS_GUARD = re.compile(r'(?:\bif\s*\([^{}]*\)|\belse)\s*(?:await\s+)?$')
|
|
|
|
|
|
def conditional_only_tests(lines):
|
|
"""A test where EVERY expect() sits inside an if/else block (so the test can end asserting nothing).
|
|
Scans `{`, `}` and `expect(` events in POSITIONAL order so a single-line `if(){expect}else{}` is
|
|
judged with the correct brace stack (not after all braces on the line are processed)."""
|
|
flagged = []
|
|
for name_i, s, e in find_test_blocks(lines):
|
|
total, conditional = 0, 0
|
|
stack = [] # frame value = True if introduced by if/else
|
|
for raw in lines[s:e + 1]:
|
|
line = strip_noise(raw)
|
|
events = sorted(
|
|
[(m.start(), m.group()) for m in re.finditer(r'[{}]', line)] +
|
|
[(m.start(), 'E') for m in EXPECT.finditer(line)]
|
|
)
|
|
for pos, kind in events:
|
|
if kind == '{':
|
|
stack.append(bool(re.search(r'\b(if|else)\b', line[:pos])))
|
|
elif kind == '}':
|
|
if stack:
|
|
stack.pop()
|
|
else: # an expect(
|
|
total += 1
|
|
# conditional if inside a braced if/else frame OR guarded by a braceless
|
|
# single-statement `if (...)` / `else` immediately before it on this line
|
|
if any(stack) or BRACELESS_GUARD.search(line[:pos]):
|
|
conditional += 1
|
|
if total > 0 and conditional == total:
|
|
flagged.append((name_i + 1, total))
|
|
return flagged
|
|
|
|
|
|
# Assertion-ish calls whose failure should surface — if these sit in a try{} whose catch swallows
|
|
# (no throw), the test passes even when the assertion fails.
|
|
ASSERT_TOKEN = re.compile(r'\bexpect\s*\(|\.waitFor\s*\(|\.toBe(Visible|Hidden|Truthy|Falsy)\b|\.toContainText\b|\.toHaveText\b')
|
|
|
|
|
|
def _match_brace(text, open_idx):
|
|
"""Index of the '}' matching the '{' at open_idx (-1 if unbalanced)."""
|
|
depth = 0
|
|
for i in range(open_idx, len(text)):
|
|
if text[i] == '{':
|
|
depth += 1
|
|
elif text[i] == '}':
|
|
depth -= 1
|
|
if depth == 0:
|
|
return i
|
|
return -1
|
|
|
|
|
|
def swallowed_assertion_tests(lines):
|
|
"""A test where an assertion sits inside a `try {}` whose matching `catch {}` does NOT re-throw —
|
|
the catch swallows the failure (logs/returns), so the test passes even when the assertion fails.
|
|
A `catch` that `throw`s, or a try/catch with no assertion inside the try (e.g. cleanup), is fine."""
|
|
flagged = []
|
|
for name_i, s, e in find_test_blocks(lines):
|
|
block = '\n'.join(strip_noise(l) for l in lines[s:e + 1])
|
|
for m in re.finditer(r'\bcatch\b\s*(\([^)]*\))?\s*\{', block):
|
|
co = block.index('{', m.start())
|
|
cc = _match_brace(block, co)
|
|
if cc < 0:
|
|
continue
|
|
if re.search(r'\bthrow\b', block[co + 1:cc]):
|
|
continue # catch re-throws → the failure surfaces → fine
|
|
pre = block[:m.start()]
|
|
tclose = pre.rstrip().rfind('}')
|
|
if tclose < 0:
|
|
continue
|
|
depth, topen = 0, -1
|
|
for i in range(tclose, -1, -1):
|
|
if pre[i] == '}':
|
|
depth += 1
|
|
elif pre[i] == '{':
|
|
depth -= 1
|
|
if depth == 0:
|
|
topen = i
|
|
break
|
|
if topen < 0 or not re.search(r'\btry\s*$', pre[:topen]):
|
|
continue
|
|
if ASSERT_TOKEN.search(pre[topen + 1:tclose]):
|
|
flagged.append(name_i + 1)
|
|
break
|
|
return flagged
|
|
|
|
|
|
def check(spec_path, baseline=None, allow_weakening=False, strengthen=False):
|
|
"""Run the static anti-pattern scan (tautologies, conditional-only assertions) and, when a
|
|
baseline is given, the diff-aware heal-weakening guard. Returns a verdict dict; never raises on
|
|
a malformed/partial baseline — missing baseline keys fall back to the current value (skip that
|
|
comparison) rather than crashing the gate.
|
|
|
|
`strengthen` (Refiner mode): the Refiner STRENGTHENS weak/mismatched tests, which can legitimately
|
|
(a) ADD negative matchers (assert a column is *absent* before a re-search) and (b) park a genuinely
|
|
unwired test as fixme. So in strengthen mode we permit a negatives rise ONLY when expect() ALSO rose
|
|
(an addition, not an in-place flip — a flip with flat expect() is still flagged), and permit a
|
|
skip/fixme rise (honest feature-incomplete). `heal-removed-assertions` (expect() may never drop) and
|
|
EVERY static anti-pattern check (tautology, conditional-only, swallowed, no-runnable) STILL apply —
|
|
so strengthen mode cannot be used to drop assertions or green-wash. (NOTE: do not also pass
|
|
--allow-weakening in strengthen mode; that disables the removed guard and defeats the point.)"""
|
|
src = open(spec_path, encoding='utf-8').read()
|
|
lines = src.split('\n')
|
|
findings = []
|
|
|
|
# --- static anti-patterns ---
|
|
for idx, raw in enumerate(lines, 1):
|
|
line = strip_noise(raw)
|
|
for pat, why in TAUTOLOGIES:
|
|
if pat.search(line):
|
|
findings.append({'severity': 'critical', 'rule': 'tautology', 'line': idx, 'detail': why})
|
|
|
|
for line_no, total in conditional_only_tests(lines):
|
|
findings.append({'severity': 'critical', 'rule': 'conditional-only-assertions', 'line': line_no,
|
|
'detail': f'all {total} expect() in this test are inside if/else — the test can pass asserting nothing'})
|
|
|
|
for line_no in swallowed_assertion_tests(lines):
|
|
findings.append({'severity': 'critical', 'rule': 'swallowed-assertion', 'line': line_no,
|
|
'detail': 'an assertion sits in a try{} whose catch swallows the failure (no throw) — the test passes even when the assertion fails'})
|
|
|
|
# No real coverage: a spec with tests where EVERY one is skip/fixme is a green run that ran
|
|
# nothing (e.g. "fixme everything" to dodge a gap). A genuinely feature-incomplete spec should
|
|
# be blocked anyway — so flag it here too. (Applies even with --allow-weakening: this is static.)
|
|
fp0 = fingerprint(src)
|
|
if fp0['tests'] > 0 and fp0['skips'] >= fp0['tests']:
|
|
findings.append({'severity': 'critical', 'rule': 'no-runnable-tests',
|
|
'detail': f"every test is skip/fixme ({fp0['skips']} skip/fixme vs {fp0['tests']} tests) — the spec asserts nothing at runtime"})
|
|
|
|
# --- diff-aware heal-weakening guard ---
|
|
fp = fingerprint(src)
|
|
if isinstance(baseline, dict) and not allow_weakening:
|
|
base_expects = baseline.get('expects', fp['expects'])
|
|
base_negatives = baseline.get('negatives', fp['negatives'])
|
|
base_skips = baseline.get('skips', fp['skips'])
|
|
if fp['expects'] < base_expects:
|
|
findings.append({'severity': 'critical', 'rule': 'heal-removed-assertions',
|
|
'detail': f"expect() count dropped {base_expects} -> {fp['expects']} during heal"})
|
|
# Inverted = a positive flipped to a negative IN PLACE. In --strengthen mode (Refiner) ADDING a
|
|
# negative is legitimate, but ONLY if expect() ALSO rose — that proves the negative is a new
|
|
# assertion, not a flip. A negative rise with no new expect() is an in-place flip → still flag,
|
|
# even in strengthen mode. (Outside strengthen mode any negative rise is flagged, as before.)
|
|
if fp['negatives'] > base_negatives and (not strengthen or fp['expects'] <= base_expects):
|
|
findings.append({'severity': 'critical', 'rule': 'heal-inverted-assertions',
|
|
'detail': f"negative matchers rose {base_negatives} -> {fp['negatives']} (positive flipped to negative in place — expect() did not rise)"})
|
|
# Rising skip/fixme is silent-skipping during heal — blocked. The Refiner's HONEST
|
|
# feature-incomplete path (park a test as fixme, assertion body kept intact) is permitted ONLY in
|
|
# --strengthen mode; expect() still cannot drop (guarded above) and an all-fixme spec is still
|
|
# caught by no-runnable-tests, so this can't be used to green-wash.
|
|
if fp['skips'] > base_skips and not strengthen:
|
|
findings.append({'severity': 'critical', 'rule': 'heal-skipped-tests',
|
|
'detail': f"skip/fixme rose {base_skips} -> {fp['skips']} during heal (use the feature-incomplete path, not silent skipping)"})
|
|
|
|
critical = [f for f in findings if f['severity'] == 'critical']
|
|
return {'verdict': 'FAIL' if critical else 'PASS', 'critical_count': len(critical),
|
|
'fingerprint': fp, 'findings': findings}
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 3 or sys.argv[1] not in ('fingerprint', 'check'):
|
|
sys.stderr.write(__doc__); sys.exit(64)
|
|
mode, spec = sys.argv[1], sys.argv[2]
|
|
if mode == 'fingerprint':
|
|
print(json.dumps(fingerprint(open(spec, encoding='utf-8').read())))
|
|
return
|
|
baseline, allow, strengthen = None, False, False
|
|
args = sys.argv[3:]
|
|
if '--baseline' in args:
|
|
baseline = json.load(open(args[args.index('--baseline') + 1]))
|
|
if '--allow-weakening' in args:
|
|
allow = True
|
|
if '--strengthen' in args:
|
|
strengthen = True
|
|
result = check(spec, baseline, allow, strengthen)
|
|
print(json.dumps(result, indent=2))
|
|
sys.exit(2 if result['verdict'] == 'FAIL' else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|