claude-math/hooks/verify-calculations.sh
boyreau f672241e9c Fix subscripted-variable false positives; finalize plugin metadata
verify-calculations.sh mistook subscripted/numbered variables (x_1, x1,
3x_1) for bare numeric equations, flagging e.g. "x_1 = 0" as the false
claim "1 = 0". The single-letter variable regex only excluded adjacent
letters, not trailing digits/underscores, and the prose segment
extractor could start a fresh match right after such a break, mining a
free digit out of the identifier.

Also carries over the author/repository metadata and the SSH-vs-HTTP
install note for the Forgejo marketplace (DISABLE_HTTP_GIT blocks the
https URL; the ssh:// path has no /forgejo/ segment).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-11-08 12:22:18 +01:00

393 lines
14 KiB
Bash
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
# Per-line verification hook for mathematical statements in the user's prompt.
#
# Streams a verdict for every verifiable line as it is checked (to /dev/tty, so
# it appears live rather than all at once when the hook returns), then emits a
# compact JSON summary on stdout so Claude sees the same results in context.
#
# Input: UserPromptSubmit hook JSON on stdin — the text lives in `.prompt`.
# Reading stdin as raw text (the previous behaviour) leaked the whole JSON
# envelope into the verdict output, so the payload is parsed properly here.
INPUT=$(cat)
[ ${#INPUT} -lt 3 ] && exit 0
export VERIFY_RAW="$INPUT"
nix-shell -p python3 python3Packages.sympy --run python3 << 'PYTHON_EOF' 2>/dev/null
import sys
import os
import re
import math
import json
import time
from sympy import isprime, totient, Symbol, Eq, solve, simplify
from sympy.parsing.sympy_parser import (
parse_expr, standard_transformations, implicit_multiplication_application)
TRANSFORMS = standard_transformations + (implicit_multiplication_application,)
raw = os.environ.get('VERIFY_RAW', '')
# The hook hands us the event JSON; fall back to raw text if that ever changes.
try:
payload = json.loads(raw)
prompt = payload.get('prompt', '') if isinstance(payload, dict) else raw
except Exception:
prompt = raw
if not prompt.strip():
sys.exit(0)
# Live channel: the controlling terminal. Absent in headless/background runs,
# where the JSON summary below becomes the only output.
try:
live = open('/dev/tty', 'w')
except Exception:
live = None
summary = []
header_done = False
def emit(text):
"""Print one line immediately, so checks appear as they happen."""
if live:
live.write(text + "\n")
live.flush()
time.sleep(0.05)
def emit_header():
global header_done
if not header_done:
emit("verifying lines...")
summary.append("verifying lines...")
header_done = True
# ----------------------------------------------------------------------
# Normalisation: course notation -> something evaluable
# ----------------------------------------------------------------------
SUPERSCRIPT = {'⁰': '0', '¹': '1', '²': '2', '³': '3',
'⁴': '4', '⁵': '5', '⁶': '6', '⁷': '7',
'⁸': '8', '⁹': '9'}
def desuperscript(s):
"""4²² -> 4^22, so superscript powers verify like ASCII ones."""
out, i = [], 0
while i < len(s):
if s[i] in SUPERSCRIPT:
run = ''
while i < len(s) and s[i] in SUPERSCRIPT:
run += SUPERSCRIPT[s[i]]
i += 1
out.append('^' + run)
else:
out.append(s[i])
i += 1
return ''.join(out)
def normalize(s):
s = desuperscript(s)
# LaTeX the user actually types
s = s.replace('\\varphi', 'phi').replace('\\phi', 'phi')
s = s.replace('\\equiv', '≡').replace('\\times', '*').replace('\\cdot', '*')
s = s.replace('\\pmod', 'mod').replace('\\mod', 'mod').replace('\\bmod', 'mod')
s = s.replace('$', '')
s = re.sub(r'\\[a-zA-Z]+', ' ', s)
# `\pmod{15}`, `8^{6}`: once the command names are gone, braces are just
# parentheses.
s = s.replace('{', '(').replace('}', ')')
# `≡ 4 (mod 15)` / `≡ 4 mod(15)` -> `≡ 4 mod 15`. The course writes the
# modulus in parentheses, and the chain scoping below needs it bare.
s = re.sub(r'\(\s*mod\s*([0-9]+)\s*\)', r'mod \1', s)
s = re.sub(r'\bmod\s*\(\s*([0-9]+)\s*\)', r'mod \1', s)
# unicode maths -> ASCII
s = s.replace('φ', 'phi').replace('ϕ', 'phi')
s = s.replace('×', '*').replace('·', '*').replace('⋅', '*')
s = s.replace('÷', '/')
s = s.replace('−', '-').replace('–', '-')
s = re.sub(r'\bpgcd\b|\bPGCD\b', 'gcd', s)
s = re.sub(r'\bppcm\b|\bPPCM\b', 'lcm', s)
s = re.sub(r'\bPhi\b', 'phi', s)
return s.strip()
ALLOWED = re.compile(r'^[0-9\s+\-*/%().,^]*$')
def _phi(n):
return int(totient(int(n)))
def _lcm(a, b):
return abs(a * b) // math.gcd(int(a), int(b)) if a and b else 0
ENV = {'phi': _phi, 'gcd': math.gcd, 'lcm': _lcm}
def eval_expr(s):
"""Evaluate a purely numeric expression (with phi/gcd/lcm) to an int."""
s = s.replace('^', '**')
s = re.sub(r'\bmod\b', '%', s) # inner mods bind like *, as in Python
for m in re.finditer(r'\*\*\s*(\d+)', s):
if int(m.group(1)) > 10000: # keep the hook from hanging
raise ValueError('exponent too large')
bare = re.sub(r'\b(phi|gcd|lcm)\b', '', s)
if not ALLOWED.match(bare):
raise ValueError('unsupported characters')
val = eval(s, {'__builtins__': {}}, dict(ENV))
if isinstance(val, float):
if abs(val - round(val)) > 1e-9:
raise ValueError('non-integer')
val = round(val)
return int(val)
MOD_TAIL = re.compile(r'^(.*)\bmod\s*([0-9]+)\s*$')
def split_mod(s):
"""`expr mod N` -> ('expr', N); no trailing modulus -> ('expr', None)."""
m = MOD_TAIL.match(s.strip())
if m:
return m.group(1).strip(), int(m.group(2))
return s.strip(), None
def eval_term(s):
"""Evaluate one side of a (chain of) equalities; a trailing `mod N`
applies to everything before it, matching the course convention."""
s = s.strip()
if not s:
raise ValueError('empty')
expr, mod = split_mod(s)
val = eval_expr(expr)
return val % mod if mod is not None else val
# Values already pinned to a variable, so a later contradictory claim is caught.
known_vars = {}
# `mod`/`phi`/`gcd`/`lcm` read as operators; any other letters end a math run,
# which is how a claim gets separated from the prose around it.
MATH_RUN = re.compile(r'(?:[0-9]|\bmod\b|\bphi\b|\bgcd\b|\blcm\b|[+\-*/^()≡=,%]|\s)+')
def extract_segments(line):
"""Pull the mathematical runs out of a line that also carries prose, so
`Euler's theorem gives 8^6 mod 15 = ... = 4 mod 15.` still gets checked
instead of being skipped because the sentence as a whole won't parse.
A letter breaks a run (so identifiers don't get read as arithmetic), but
a digit/underscore right after it doesn't - so for `x_1 = 0` the `_`
broke the run and the following `1` started a *new* one, mining the
bogus standalone claim `1 = 0` out of a variable name. A run that starts
right after a letter/digit/underscore is such a fragment, chopped out of
a longer identifier, not a real standalone number - discard it instead
of checking it."""
segments = []
for m in MATH_RUN.finditer(line):
start = m.start()
if start > 0 and re.match(r'[a-zA-Z0-9_]', line[start - 1]):
continue
seg = m.group(0).strip(' ,;:.')
if not seg or not re.search(r'≡|=', seg):
continue
if len([p for p in re.split(r'≡|=', seg) if p.strip()]) >= 2:
segments.append(seg)
return segments
def verify_claim(line):
"""-> (status, detail) for one already-normalised claim."""
if not line or len(line) < 3:
return ('skip', None)
# --- primality: "13 is prime", "15 n'est pas premier" ---------------
m = re.search(r'\b([0-9]+)\s+(?:is|est)\s+(not\s+|n\'est\s+pas\s+|pas\s+)?'
r'(?:a\s+|un\s+|une\s+)?(?:prime|premier)', line, re.I)
if not m:
m2 = re.search(r'\b([0-9]+)\s+n\'est\s+pas\s+premier', line, re.I)
if m2:
n = int(m2.group(1))
actual = bool(isprime(n))
return ('ok' if not actual else 'wrong',
f"{n} est premier" if actual else None)
if m:
n = int(m.group(1))
claims_not = bool(m.group(2))
actual = bool(isprime(n))
ok = (actual != claims_not)
detail = None if ok else (f"{n} est premier" if actual
else f"{n} = composé")
return ('ok' if ok else 'wrong', detail)
# --- divisibility: "7 | 28", "7 | (45 - 17)" ------------------------
if '|' in line and line.count('|') == 1 and '||' not in line:
left, right = line.split('|')
try:
d, n = eval_term(left), eval_term(right)
except Exception:
pass
else:
if d == 0:
return ('wrong', 'division par 0')
ok = (n % d == 0)
return ('ok' if ok else 'wrong',
None if ok else f"reste {n % d}")
# --- chain of equalities / congruences ------------------------------
# A `mod N` written on ANY link scopes the WHOLE chain. The course states
# the modulus once and leaves it implicit everywhere else, so
# 8^6 mod 15 = 64*8^4 = 4*8*8^3 = 2*8*8^2 = 1*8^2 = 4
# is read entirely modulo 15: every link gets reduced by 15, not just the
# one that happens to spell the modulus out. Scoping only the trailing
# `mod N` reported correct chains as wrong whenever the modulus was
# written on the first link instead of the last.
parts, mods = [], []
for p in re.split(r'≡|=', line):
p = p.strip()
if not p:
continue
expr, n = split_mod(p)
parts.append(expr if expr else p)
if n is not None:
mods.append(n)
distinct = sorted(set(mods))
# Two different moduli in one chain is not a statement we can read with
# confidence; stay silent rather than accuse wrongly.
if len(distinct) > 1:
return ('skip', None)
mod = distinct[0] if distinct else None
body = ' = '.join(parts)
if len(parts) >= 2:
try:
vals = [eval_expr(p) for p in parts]
except Exception:
vals = None
if vals is not None:
red = [v % mod for v in vals] if mod is not None else vals
ok = all(r == red[0] for r in red)
if ok:
return ('ok', None)
shown = ', '.join(str(r) for r in red)
return ('wrong', f"{shown} mod {mod}" if mod is not None else shown)
# --- symbolic fallback: one unknown, e.g. "2x + 3 = 7" ----------
# Skipped when the chain carries a modulus: solving `x ≡ 2 mod 5` as
# the equation x = 2 would drop the modulus and report a verification
# that never happened.
if mod is not None:
return ('skip', None)
# Prose sentences must not reach sympy: `parse_expr` with implicit
# multiplication turns every letter into a symbol, which is both
# meaningless and slow enough to blow the hook's timeout.
if re.search(r'[a-zA-Z]{2,}', re.sub(r'\b(mod|phi|gcd|lcm)\b', '', body)):
return ('skip', None)
# A trailing digit/underscore must also break the match (not just a
# trailing letter), so a subscripted name like `x_1` or `x1` isn't
# mistaken for the lone variable `x` with its subscript silently
# dropped. A LEADING digit is fine and must stay allowed: `2x` is
# ordinary coefficient notation for `2*x`.
if len(parts) == 2 and re.search(r'(?<![a-zA-Z])[a-zA-Z](?![a-zA-Z0-9_])', body):
var = re.search(r'(?<![a-zA-Z])[a-zA-Z](?![a-zA-Z0-9_])', body).group(0)
if var in ('e', 'I', 'N', 'S', 'O'):
return ('skip', None)
try:
x = Symbol(var)
lhs = parse_expr(parts[0].replace('^', '**'),
transformations=TRANSFORMS, local_dict={var: x})
rhs = parse_expr(parts[1].replace('^', '**'),
transformations=TRANSFORMS, local_dict={var: x})
sols = solve(Eq(lhs, rhs), x)
except Exception:
return ('skip', None)
if not sols:
return ('skip', None)
if len(sols) == 1 and sols[0].is_number:
prev = known_vars.get(var)
if prev is not None and simplify(sols[0] - prev) != 0:
return ('wrong', f"{var} = {sols[0]} contredit {var} = {prev}")
known_vars[var] = sols[0]
return ('ok', f"{var} = {', '.join(str(s) for s in sols)}")
return ('skip', None)
def verify_line(raw_line):
"""-> list of (shown_text, status, detail) for one source line.
A line is first tried whole; only if that yields nothing do we fall back
to the maths embedded in prose, which may hold several claims at once
(`phi(15) = 8; 8^6 mod 15 = ... = 4 mod 15.`)."""
line = normalize(raw_line)
if not line:
return []
status, detail = verify_claim(line)
if status != 'skip':
return [(raw_line.strip(), status, detail)]
results = []
for seg in extract_segments(line):
s, d = verify_claim(seg)
if s != 'skip':
results.append((seg.strip(), s, d))
return results
# ----------------------------------------------------------------------
# Walk the prompt line by line, printing each verdict as it is decided.
# ----------------------------------------------------------------------
lines = prompt.split('\n')
last_idx = -2
for i, raw_line in enumerate(lines):
try:
verdicts = verify_line(raw_line)
except Exception:
verdicts = []
if not verdicts:
continue
emit_header()
# First line of a new block: blank line, then the line above it for context
# (typically the exercise label or the statement being worked on).
if i - 1 != last_idx:
emit("")
summary.append("")
if i > 0 and lines[i - 1].strip():
ctx = lines[i - 1].strip()
emit(ctx)
summary.append(ctx)
for shown, status, detail in verdicts:
verdict = f"{shown} = Ok ✅" if status == 'ok' else f"{shown} = Wrong ❌"
if detail:
verdict += f" ({detail})"
emit(verdict)
summary.append(verdict)
last_idx = i
if live:
try:
live.close()
except Exception:
pass
# Give Claude the same verdicts in context (and cover the no-tty case).
if any(s.strip() for s in summary):
print(json.dumps({"systemMessage": "\n".join(summary).strip()}))
PYTHON_EOF