claude-math/hooks/verify-calculations.sh

393 lines
14 KiB
Bash
Raw Normal View History

#!/bin/bash
# Per-line verification hook for mathematical statements in the user's prompt.
#
# Streams a verdict for every verifiable line as it is checked (to /dev/tty, so
# it appears live rather than all at once when the hook returns), then emits a
# compact JSON summary on stdout so Claude sees the same results in context.
#
# Input: UserPromptSubmit hook JSON on stdin — the text lives in `.prompt`.
# Reading stdin as raw text (the previous behaviour) leaked the whole JSON
# envelope into the verdict output, so the payload is parsed properly here.
INPUT=$(cat)
[ ${#INPUT} -lt 3 ] && exit 0
export VERIFY_RAW="$INPUT"
nix-shell -p python3 python3Packages.sympy --run python3 << 'PYTHON_EOF' 2>/dev/null
import sys
import os
import re
import math
import json
import time
from sympy import isprime, totient, Symbol, Eq, solve, simplify
from sympy.parsing.sympy_parser import (
parse_expr, standard_transformations, implicit_multiplication_application)
TRANSFORMS = standard_transformations + (implicit_multiplication_application,)
raw = os.environ.get('VERIFY_RAW', '')
# The hook hands us the event JSON; fall back to raw text if that ever changes.
try:
payload = json.loads(raw)
prompt = payload.get('prompt', '') if isinstance(payload, dict) else raw
except Exception:
prompt = raw
if not prompt.strip():
sys.exit(0)
# Live channel: the controlling terminal. Absent in headless/background runs,
# where the JSON summary below becomes the only output.
try:
live = open('/dev/tty', 'w')
except Exception:
live = None
summary = []
header_done = False
def emit(text):
"""Print one line immediately, so checks appear as they happen."""
if live:
live.write(text + "\n")
live.flush()
time.sleep(0.05)
def emit_header():
global header_done
if not header_done:
emit("verifying lines...")
summary.append("verifying lines...")
header_done = True
# ----------------------------------------------------------------------
# Normalisation: course notation -> something evaluable
# ----------------------------------------------------------------------
SUPERSCRIPT = {'⁰': '0', '¹': '1', '²': '2', '³': '3',
'⁴': '4', '⁵': '5', '⁶': '6', '⁷': '7',
'⁸': '8', '⁹': '9'}
def desuperscript(s):
"""4²² -> 4^22, so superscript powers verify like ASCII ones."""
out, i = [], 0
while i < len(s):
if s[i] in SUPERSCRIPT:
run = ''
while i < len(s) and s[i] in SUPERSCRIPT:
run += SUPERSCRIPT[s[i]]
i += 1
out.append('^' + run)
else:
out.append(s[i])
i += 1
return ''.join(out)
def normalize(s):
s = desuperscript(s)
# LaTeX the user actually types
s = s.replace('\\varphi', 'phi').replace('\\phi', 'phi')
s = s.replace('\\equiv', '≡').replace('\\times', '*').replace('\\cdot', '*')
s = s.replace('\\pmod', 'mod').replace('\\mod', 'mod').replace('\\bmod', 'mod')
s = s.replace('$', '')
s = re.sub(r'\\[a-zA-Z]+', ' ', s)
# `\pmod{15}`, `8^{6}`: once the command names are gone, braces are just
# parentheses.
s = s.replace('{', '(').replace('}', ')')
# `≡ 4 (mod 15)` / `≡ 4 mod(15)` -> `≡ 4 mod 15`. The course writes the
# modulus in parentheses, and the chain scoping below needs it bare.
s = re.sub(r'\(\s*mod\s*([0-9]+)\s*\)', r'mod \1', s)
s = re.sub(r'\bmod\s*\(\s*([0-9]+)\s*\)', r'mod \1', s)
# unicode maths -> ASCII
s = s.replace('φ', 'phi').replace('ϕ', 'phi')
s = s.replace('×', '*').replace('·', '*').replace('⋅', '*')
s = s.replace('÷', '/')
s = s.replace('−', '-').replace('–', '-')
s = re.sub(r'\bpgcd\b|\bPGCD\b', 'gcd', s)
s = re.sub(r'\bppcm\b|\bPPCM\b', 'lcm', s)
s = re.sub(r'\bPhi\b', 'phi', s)
return s.strip()
ALLOWED = re.compile(r'^[0-9\s+\-*/%().,^]*$')
def _phi(n):
return int(totient(int(n)))
def _lcm(a, b):
return abs(a * b) // math.gcd(int(a), int(b)) if a and b else 0
ENV = {'phi': _phi, 'gcd': math.gcd, 'lcm': _lcm}
def eval_expr(s):
"""Evaluate a purely numeric expression (with phi/gcd/lcm) to an int."""
s = s.replace('^', '**')
s = re.sub(r'\bmod\b', '%', s) # inner mods bind like *, as in Python
for m in re.finditer(r'\*\*\s*(\d+)', s):
if int(m.group(1)) > 10000: # keep the hook from hanging
raise ValueError('exponent too large')
bare = re.sub(r'\b(phi|gcd|lcm)\b', '', s)
if not ALLOWED.match(bare):
raise ValueError('unsupported characters')
val = eval(s, {'__builtins__': {}}, dict(ENV))
if isinstance(val, float):
if abs(val - round(val)) > 1e-9:
raise ValueError('non-integer')
val = round(val)
return int(val)
MOD_TAIL = re.compile(r'^(.*)\bmod\s*([0-9]+)\s*$')
def split_mod(s):
"""`expr mod N` -> ('expr', N); no trailing modulus -> ('expr', None)."""
m = MOD_TAIL.match(s.strip())
if m:
return m.group(1).strip(), int(m.group(2))
return s.strip(), None
def eval_term(s):
"""Evaluate one side of a (chain of) equalities; a trailing `mod N`
applies to everything before it, matching the course convention."""
s = s.strip()
if not s:
raise ValueError('empty')
expr, mod = split_mod(s)
val = eval_expr(expr)
return val % mod if mod is not None else val
# Values already pinned to a variable, so a later contradictory claim is caught.
known_vars = {}
# `mod`/`phi`/`gcd`/`lcm` read as operators; any other letters end a math run,
# which is how a claim gets separated from the prose around it.
MATH_RUN = re.compile(r'(?:[0-9]|\bmod\b|\bphi\b|\bgcd\b|\blcm\b|[+\-*/^()≡=,%]|\s)+')
def extract_segments(line):
"""Pull the mathematical runs out of a line that also carries prose, so
`Euler's theorem gives 8^6 mod 15 = ... = 4 mod 15.` still gets checked
instead of being skipped because the sentence as a whole won't parse.
A letter breaks a run (so identifiers don't get read as arithmetic), but
a digit/underscore right after it doesn't - so for `x_1 = 0` the `_`
broke the run and the following `1` started a *new* one, mining the
bogus standalone claim `1 = 0` out of a variable name. A run that starts
right after a letter/digit/underscore is such a fragment, chopped out of
a longer identifier, not a real standalone number - discard it instead
of checking it."""
segments = []
for m in MATH_RUN.finditer(line):
start = m.start()
if start > 0 and re.match(r'[a-zA-Z0-9_]', line[start - 1]):
continue
seg = m.group(0).strip(' ,;:.')
if not seg or not re.search(r'≡|=', seg):
continue
if len([p for p in re.split(r'≡|=', seg) if p.strip()]) >= 2:
segments.append(seg)
return segments
def verify_claim(line):
"""-> (status, detail) for one already-normalised claim."""
if not line or len(line) < 3:
return ('skip', None)
# --- primality: "13 is prime", "15 n'est pas premier" ---------------
m = re.search(r'\b([0-9]+)\s+(?:is|est)\s+(not\s+|n\'est\s+pas\s+|pas\s+)?'
r'(?:a\s+|un\s+|une\s+)?(?:prime|premier)', line, re.I)
if not m:
m2 = re.search(r'\b([0-9]+)\s+n\'est\s+pas\s+premier', line, re.I)
if m2:
n = int(m2.group(1))
actual = bool(isprime(n))
return ('ok' if not actual else 'wrong',
f"{n} est premier" if actual else None)
if m:
n = int(m.group(1))
claims_not = bool(m.group(2))
actual = bool(isprime(n))
ok = (actual != claims_not)
detail = None if ok else (f"{n} est premier" if actual
else f"{n} = composé")
return ('ok' if ok else 'wrong', detail)
# --- divisibility: "7 | 28", "7 | (45 - 17)" ------------------------
if '|' in line and line.count('|') == 1 and '||' not in line:
left, right = line.split('|')
try:
d, n = eval_term(left), eval_term(right)
except Exception:
pass
else:
if d == 0:
return ('wrong', 'division par 0')
ok = (n % d == 0)
return ('ok' if ok else 'wrong',
None if ok else f"reste {n % d}")
# --- chain of equalities / congruences ------------------------------
# A `mod N` written on ANY link scopes the WHOLE chain. The course states
# the modulus once and leaves it implicit everywhere else, so
# 8^6 mod 15 = 64*8^4 = 4*8*8^3 = 2*8*8^2 = 1*8^2 = 4
# is read entirely modulo 15: every link gets reduced by 15, not just the
# one that happens to spell the modulus out. Scoping only the trailing
# `mod N` reported correct chains as wrong whenever the modulus was
# written on the first link instead of the last.
parts, mods = [], []
for p in re.split(r'≡|=', line):
p = p.strip()
if not p:
continue
expr, n = split_mod(p)
parts.append(expr if expr else p)
if n is not None:
mods.append(n)
distinct = sorted(set(mods))
# Two different moduli in one chain is not a statement we can read with
# confidence; stay silent rather than accuse wrongly.
if len(distinct) > 1:
return ('skip', None)
mod = distinct[0] if distinct else None
body = ' = '.join(parts)
if len(parts) >= 2:
try:
vals = [eval_expr(p) for p in parts]
except Exception:
vals = None
if vals is not None:
red = [v % mod for v in vals] if mod is not None else vals
ok = all(r == red[0] for r in red)
if ok:
return ('ok', None)
shown = ', '.join(str(r) for r in red)
return ('wrong', f"{shown} mod {mod}" if mod is not None else shown)
# --- symbolic fallback: one unknown, e.g. "2x + 3 = 7" ----------
# Skipped when the chain carries a modulus: solving `x ≡ 2 mod 5` as
# the equation x = 2 would drop the modulus and report a verification
# that never happened.
if mod is not None:
return ('skip', None)
# Prose sentences must not reach sympy: `parse_expr` with implicit
# multiplication turns every letter into a symbol, which is both
# meaningless and slow enough to blow the hook's timeout.
if re.search(r'[a-zA-Z]{2,}', re.sub(r'\b(mod|phi|gcd|lcm)\b', '', body)):
return ('skip', None)
# A trailing digit/underscore must also break the match (not just a
# trailing letter), so a subscripted name like `x_1` or `x1` isn't
# mistaken for the lone variable `x` with its subscript silently
# dropped. A LEADING digit is fine and must stay allowed: `2x` is
# ordinary coefficient notation for `2*x`.
if len(parts) == 2 and re.search(r'(?<![a-zA-Z])[a-zA-Z](?![a-zA-Z0-9_])', body):
var = re.search(r'(?<![a-zA-Z])[a-zA-Z](?![a-zA-Z0-9_])', body).group(0)
if var in ('e', 'I', 'N', 'S', 'O'):
return ('skip', None)
try:
x = Symbol(var)
lhs = parse_expr(parts[0].replace('^', '**'),
transformations=TRANSFORMS, local_dict={var: x})
rhs = parse_expr(parts[1].replace('^', '**'),
transformations=TRANSFORMS, local_dict={var: x})
sols = solve(Eq(lhs, rhs), x)
except Exception:
return ('skip', None)
if not sols:
return ('skip', None)
if len(sols) == 1 and sols[0].is_number:
prev = known_vars.get(var)
if prev is not None and simplify(sols[0] - prev) != 0:
return ('wrong', f"{var} = {sols[0]} contredit {var} = {prev}")
known_vars[var] = sols[0]
return ('ok', f"{var} = {', '.join(str(s) for s in sols)}")
return ('skip', None)
def verify_line(raw_line):
"""-> list of (shown_text, status, detail) for one source line.
A line is first tried whole; only if that yields nothing do we fall back
to the maths embedded in prose, which may hold several claims at once
(`phi(15) = 8; 8^6 mod 15 = ... = 4 mod 15.`)."""
line = normalize(raw_line)
if not line:
return []
status, detail = verify_claim(line)
if status != 'skip':
return [(raw_line.strip(), status, detail)]
results = []
for seg in extract_segments(line):
s, d = verify_claim(seg)
if s != 'skip':
results.append((seg.strip(), s, d))
return results
# ----------------------------------------------------------------------
# Walk the prompt line by line, printing each verdict as it is decided.
# ----------------------------------------------------------------------
lines = prompt.split('\n')
last_idx = -2
for i, raw_line in enumerate(lines):
try:
verdicts = verify_line(raw_line)
except Exception:
verdicts = []
if not verdicts:
continue
emit_header()
# First line of a new block: blank line, then the line above it for context
# (typically the exercise label or the statement being worked on).
if i - 1 != last_idx:
emit("")
summary.append("")
if i > 0 and lines[i - 1].strip():
ctx = lines[i - 1].strip()
emit(ctx)
summary.append(ctx)
for shown, status, detail in verdicts:
verdict = f"{shown} = Ok ✅" if status == 'ok' else f"{shown} = Wrong ❌"
if detail:
verdict += f" ({detail})"
emit(verdict)
summary.append(verdict)
last_idx = i
if live:
try:
live.close()
except Exception:
pass
# Give Claude the same verdicts in context (and cover the no-tty case).
if any(s.strip() for s in summary):
print(json.dumps({"systemMessage": "\n".join(summary).strip()}))
PYTHON_EOF