unslop — Skillopedia

Unslop Humanize AI-generated prose. Audit first. Rewrite only when the user asks for a rewrite. For every audit or rewrite, read references/core-contract.md. It is the single behavior contract. Command files define routing and mechanics; presets supply optional voice, but neither can override the core contract. Routing When the user invokes a sub-command ( , ), you MUST read before acting. Non-optional — the command file defines the flow, and skipping it drops steps the user expects. A bare with no leading command word defaults to . If the first word does not match a command but the intent cl…

, part):\n current += part\n sentences.append(current.strip())\n current = \"\"\n else:\n current += part\n if current.strip():\n sentences.append(current.strip())\n # Filter empty and clean up\n return [s.strip() for s in sentences if s.strip()]\n\n\ndef split_staccato_units(text: str) -> list[str]:\n units = re.split(r'\\s*[—;]\\s*|(?\u003c=[.!?])\\s+', text)\n return [u.strip() for u in units if u.strip()]\n\n\ndef split_words(text: str) -> list[str]:\n \"\"\"Split text into words.\"\"\"\n # Remove punctuation except hyphens in words. Numeric tokens are real words\n # (a data table is not \"empty text\"), so keep them.\n text = re.sub(r'[^\\w\\s-]', ' ', text)\n words = text.lower().split()\n return [w for w in words if w]\n\n\ndef calculate_metrics(text: str) -> ReadabilityMetrics:\n \"\"\"Calculate all readability metrics.\"\"\"\n sentences = split_sentences(text)\n words = split_words(text)\n paragraphs = [p.strip() for p in text.split('\\n\\n') if p.strip()]\n\n if not sentences or not words:\n return {\n \"flesch_kincaid_grade\": 0,\n \"flesch_reading_ease\": 0,\n \"sentence_count\": 0,\n \"word_count\": 0,\n \"avg_sentence_length\": 0,\n \"sentence_length_variance\": 0,\n \"min_sentence_length\": 0,\n \"max_sentence_length\": 0,\n \"consecutive_similar_length\": 0,\n \"word_repetition_score\": 0,\n \"top_repeated_words\": [],\n \"paragraph_count\": len(paragraphs),\n \"avg_paragraph_length\": 0,\n \"flags\": [\"Empty or invalid text\"]\n }\n\n # Basic counts\n sentence_count = len(sentences)\n word_count = len(words)\n syllable_count = sum(count_syllables(w) for w in words)\n\n # Sentence lengths\n sentence_lengths = [len(split_words(s)) for s in sentences]\n avg_sentence_length = word_count / sentence_count if sentence_count else 0\n\n # Variance calculation\n if len(sentence_lengths) > 1:\n mean = sum(sentence_lengths) / len(sentence_lengths)\n variance = sum((x - mean) ** 2 for x in sentence_lengths) / len(sentence_lengths)\n else:\n variance = 0\n\n # Consecutive similar length sentences\n consecutive_similar = 0\n max_consecutive = 0\n for i in range(1, len(sentence_lengths)):\n # \"Similar\" = within 3 words of each other\n if abs(sentence_lengths[i] - sentence_lengths[i-1]) \u003c= 3:\n consecutive_similar += 1\n max_consecutive = max(max_consecutive, consecutive_similar)\n else:\n consecutive_similar = 0\n\n # Longest run of tiny (\u003c=5-word) sentence-like units — the staccato \"anti-slop\" cadence.\n # Tracked independently of sentence_count so short de-slopped outputs don't slip.\n staccato_lengths = [len(split_words(s)) for s in split_staccato_units(text)]\n staccato_run = 0\n max_staccato_run = 0\n for length in staccato_lengths:\n if length \u003c= 5:\n staccato_run += 1\n max_staccato_run = max(max_staccato_run, staccato_run)\n else:\n staccato_run = 0\n\n # Flesch-Kincaid calculations\n avg_syllables_per_word = syllable_count / word_count if word_count else 0\n\n flesch_reading_ease = (\n 206.835\n - (1.015 * avg_sentence_length)\n - (84.6 * avg_syllables_per_word)\n )\n\n flesch_kincaid_grade = (\n (0.39 * avg_sentence_length)\n + (11.8 * avg_syllables_per_word)\n - 15.59\n )\n\n # Word repetition (excluding common words)\n stop_words = {\n 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',\n 'of', 'with', 'by', 'from', 'is', 'are', 'was', 'were', 'be', 'been',\n 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',\n 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',\n 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',\n 'we', 'they', 'what', 'which', 'who', 'when', 'where', 'why', 'how',\n 'not', 'no', 'yes', 'if', 'then', 'else', 'so', 'as', 'than', 'just'\n }\n\n content_words = [w for w in words if w not in stop_words and len(w) > 2]\n word_freq = Counter(content_words)\n\n # Repetition score: percentage of content words appearing 3+ times\n repeated_words = {w: c for w, c in word_freq.items() if c >= 3}\n repetition_score = (\n sum(repeated_words.values()) / len(content_words) * 100\n if content_words else 0\n )\n\n top_repeated = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:10]\n\n # Paragraph stats\n para_lengths = [len(split_words(p)) for p in paragraphs]\n avg_para_length = sum(para_lengths) / len(para_lengths) if para_lengths else 0\n\n # Generate flags\n flags: list[str] = []\n\n if flesch_kincaid_grade > 12:\n flags.append(f\"High reading level ({flesch_kincaid_grade:.1f} grade)\")\n\n if variance \u003c 10 and sentence_count > 3:\n flags.append(\"Low sentence length variance (monotonous rhythm)\")\n\n if max_consecutive >= 3:\n flags.append(f\"{max_consecutive}+ consecutive similar-length sentences\")\n\n if repetition_score > 15:\n flags.append(f\"High word repetition ({repetition_score:.1f}%)\")\n\n if avg_sentence_length > 25:\n flags.append(f\"Long average sentence length ({avg_sentence_length:.1f} words)\")\n\n if avg_sentence_length \u003c 8 and sentence_count > 3:\n flags.append(\"Very short sentences (may feel choppy)\")\n\n # Staccato cadence is the loudest \"anti-slop AI\" signature; flag it even on\n # short outputs (no sentence_count gate) so de-slopped text can't hide it.\n if max_staccato_run >= 3:\n flags.append(f\"Staccato cadence: {max_staccato_run} consecutive tiny sentences (anti-slop AI tell)\")\n elif avg_sentence_length \u003c 6 and sentence_count >= 2:\n flags.append(f\"Staccato cadence (avg {avg_sentence_length:.1f} words/sentence — anti-slop AI tell)\")\n\n if max(sentence_lengths) - min(sentence_lengths) \u003c 5 and sentence_count > 5:\n flags.append(\"Sentences all similar length (AI tell)\")\n\n return {\n \"flesch_kincaid_grade\": round(flesch_kincaid_grade, 1),\n \"flesch_reading_ease\": round(flesch_reading_ease, 1),\n \"sentence_count\": sentence_count,\n \"word_count\": word_count,\n \"avg_sentence_length\": round(avg_sentence_length, 1),\n \"sentence_length_variance\": round(variance, 1),\n \"min_sentence_length\": min(sentence_lengths) if sentence_lengths else 0,\n \"max_sentence_length\": max(sentence_lengths) if sentence_lengths else 0,\n \"consecutive_similar_length\": max_consecutive,\n \"word_repetition_score\": round(repetition_score, 1),\n \"top_repeated_words\": top_repeated,\n \"paragraph_count\": len(paragraphs),\n \"avg_paragraph_length\": round(avg_para_length, 1),\n \"flags\": flags\n }\n\n\ndef parse_args(argv: list[str]) -> argparse.Namespace:\n parser = argparse.ArgumentParser(\n description=\"Calculate readability metrics for transformed text.\"\n )\n parser.add_argument(\"path\", nargs=\"?\", help=\"Path to input text file (default: read stdin)\")\n return parser.parse_args(argv)\n\n\ndef main() -> None:\n args = parse_args(sys.argv[1:])\n\n # Read input\n if args.path:\n try:\n with open(args.path, 'r', errors=\"replace\") as f:\n text = f.read()\n except OSError as e:\n print(json.dumps({\"error\": f\"Could not read input: {e}\"}))\n sys.exit(2)\n else:\n text = sys.stdin.buffer.read().decode(\"utf-8\", errors=\"replace\")\n\n if not text.strip():\n print(json.dumps({\"error\": \"No input provided\"}))\n sys.exit(1)\n\n metrics = calculate_metrics(text)\n print(json.dumps(metrics, indent=2))\n\n # Exit with 1 if any flags (warnings)\n sys.exit(1 if metrics[\"flags\"] else 0)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":9772,"content_sha256":"aae683df79680b193280299b08ef972f73ac7615a5d6a739eae5b37f25f1b9b9"},{"filename":"scripts/refresh_status.py","content":"#!/usr/bin/env python3\n\"\"\"\nReport staleness of the three inputs the adversarial-refresh promise depends\non: the Wikipedia wiki-sync state, the model-parity bench, and the newest\nadversarial-eval row. Pure stdlib, network-free, always exits 0 — this is a\nreporter an agent reads before deciding what to run next\n(see references/refresh.md), not a gate that blocks anything itself.\n\nUsage:\n python3 scripts/refresh_status.py\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport re\nimport subprocess\nimport sys\nfrom datetime import date, datetime, timezone\nfrom pathlib import Path\n\nROOT = Path(__file__).resolve().parent.parent\nWIKI_STATE_FILE = ROOT / \"scripts\" / \".wiki_sync_state.json\"\nPIPELINE_DOC = ROOT / \"references\" / \"pipeline.md\"\nADVERSARIAL_EVALS = ROOT / \"evals\" / \"adversarial-evals.json\"\n\n# Guidance thresholds, not hard limits -- the numbers below are the point at\n# which an agent should proactively check, not a required cadence. Every\n# field is a plain reporter value; nothing here fails the build.\n#\n# Wikipedia's \"Signs of AI writing\" page is a slow-moving reference page;\n# quarterly is enough to catch drift without chasing noise.\nWIKI_STALE_DAYS = 90\n# The GPT/Anthropic model spectrums shift on the order of months, and the\n# rule in references/pipeline.md already forces a re-run on any co-writer,\n# mimic, or detector-pack change regardless of this threshold -- this is a\n# backstop for the case where no such change happened but time still passed.\nBENCH_STALE_DAYS = 180\n# The adversarial-eval catalog is the product's growth signal (docs/PRODUCT.md,\n# Growth); a two-month gap with no new row is worth a look even if nothing\n# else prompted one.\nROW_STALE_DAYS = 60\n\n# The convention this script relies on in references/pipeline.md: the\n# recorded parity date is written as \"Live matrix recorded **YYYY-MM-DD**.\"\n# If that sentence's wording changes, this regex needs to change with it.\nPIPELINE_DATE_RE = re.compile(\n r\"Live matrix recorded \\*\\*(\\d{4}-\\d{2}-\\d{2})\\*\\*\"\n)\n\n\ndef _today() -> date:\n return datetime.now(timezone.utc).date()\n\n\ndef _days_since(d: date | None) -> int | None:\n if d is None:\n return None\n return (_today() - d).days\n\n\ndef _status(last: date | None, stale_days: int) -> dict:\n days = _days_since(last)\n stale = days is None or days > stale_days\n return {\n \"last\": last.isoformat() if last else None,\n \"days\": days,\n \"stale\": stale,\n }\n\n\ndef _parse_iso_date(value: str) -> date | None:\n \"\"\"Best-effort parse of an ISO-8601 timestamp (MediaWiki style, 'Z'\n suffix included) down to a plain date.\"\"\"\n try:\n return datetime.fromisoformat(value.replace(\"Z\", \"+00:00\")).date()\n except (ValueError, AttributeError):\n return None\n\n\ndef wiki_sync_status() -> dict:\n if not WIKI_STATE_FILE.exists():\n # Never synced. Treat as stale so it surfaces for attention.\n return _status(None, WIKI_STALE_DAYS)\n\n last: date | None = None\n try:\n state = json.loads(WIKI_STATE_FILE.read_text())\n last = _parse_iso_date(state.get(\"last_timestamp\", \"\"))\n except (json.JSONDecodeError, OSError):\n last = None\n\n if last is None:\n # Content didn't parse; fall back to the state file's mtime.\n try:\n mtime = WIKI_STATE_FILE.stat().st_mtime\n last = datetime.fromtimestamp(mtime, tz=timezone.utc).date()\n except OSError:\n last = None\n\n return _status(last, WIKI_STALE_DAYS)\n\n\ndef parity_bench_status() -> dict:\n last: date | None = None\n try:\n text = PIPELINE_DOC.read_text()\n match = PIPELINE_DATE_RE.search(text)\n if match:\n last = date.fromisoformat(match.group(1))\n except OSError:\n last = None\n\n return _status(last, BENCH_STALE_DAYS)\n\n\ndef newest_pattern_row_status() -> dict:\n last: date | None = None\n try:\n result = subprocess.run(\n [\n \"git\",\n \"log\",\n \"-1\",\n \"--format=%cs\",\n \"--\",\n str(ADVERSARIAL_EVALS),\n ],\n cwd=str(ROOT),\n capture_output=True,\n text=True,\n timeout=10,\n )\n out = result.stdout.strip()\n if result.returncode == 0 and out:\n last = date.fromisoformat(out)\n except (OSError, subprocess.SubprocessError, ValueError):\n last = None\n\n return _status(last, ROW_STALE_DAYS)\n\n\ndef main() -> None:\n output = {\n \"wiki_sync\": wiki_sync_status(),\n \"parity_bench\": parity_bench_status(),\n \"newest_pattern_row\": newest_pattern_row_status(),\n }\n print(json.dumps(output, indent=2))\n sys.exit(0)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":4766,"content_sha256":"aa924d592f6692a9ad3a01136a51019fcac0c067f0bdd648a5ac1cf80bd89160"},{"filename":"scripts/silhouette_scan.py","content":"#!/usr/bin/env python3\n\"\"\"Scan prose for discourse-level silhouette AI-writing patterns.\n\nThis is the macro-structure scanner one level above scripts/structure_scan.py.\nstructure_scan measures SURFACE structure (sentence cadence, opener words,\nlisticle formatting). silhouette_scan measures IDEA ARRANGEMENT: does the\ndocument follow a template-shaped outline, preview-then-fulfill its own points,\nopen body paragraphs with rotating discourse cues, and close with a recap loop?\n\nFive validated one-sided tells (all AI-high; humans cluster at zero):\n\n scaffold_opener_share body paragraphs opening with a discourse-cue class\n callback_content early vocab absent mid-document, returning at the end\n (the recap loop; strongest single tell)\n role_entropy_bits count of distinct cue-opener classes (a template rotates\n \"However / In addition / Ultimately\" openers)\n preview_fulfillment intro content words reappearing as body-paragraph heads\n heading_preview heading head-nouns previewed in the intro (outline follow)\n\nComposite:\n\n silhouette_penalty = sum_i weight_i * relu((m_i - median_i) / scale_i)\n\nscored against a committed HUMAN reference distribution\n(evals/fixtures/silhouette/human_reference.json), negative side clipped. The\ndocument flags when silhouette_penalty >= 1.0.\n\nScale note (deviation from a naive sample-IQR denominator, stated on purpose):\nthese five metrics are degenerate at zero across the human corpus -- the sample\nIQR is 0.0 for every one, so the literal (m - median)/IQR with a 0.05 floor\nreduces to m/0.05 and produces human false positives (a human doc with one\ngenuine callback at 0.17 lands at penalty > 1.0). The reference therefore scales\neach metric by its human UPPER FENCE -- the validated activation threshold past\nwhich a human essentially never scores. The scorer uses\ndenom = max(sample_iqr, fence) so a future per-author profile with a real,\nnon-degenerate IQR widens the scale naturally. This keeps the exact weighted\nrelu-of-z shape, reproduces the research's struct01/03/09/11 signature, and holds\n0 / 8 human false positives on the validation corpus.\n\nVoice fingerprint (not implemented here): the same per-metric median/scale\nmachinery generalizes to a PER-AUTHOR reference stored in a voice profile, so\nmimic mode can penalize |draft - author-median| instead of generic-human. That\nper-author integration lands with the teach/mimic branch (WP10b); this scanner\nships the generic-human reference only.\n\nstdlib only. JSON to stdout. Exit 1 on flag, 0 clean, 2 on a missing file.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport math\nimport re\nimport sys\nfrom collections import Counter\nfrom pathlib import Path\n\nHERE = Path(__file__).resolve().parent\nsys.path.insert(0, str(HERE))\n\nfrom structure_scan import STOPWORDS as _STRUCTURE_STOPWORDS # noqa: E402\nfrom _lang import ( # noqa: E402\n is_probably_english,\n paragraphs as _prose_paragraphs,\n words,\n)\n\n\nREFERENCE_PATH = (\n Path(__file__).resolve().parent.parent\n / \"evals\" / \"fixtures\" / \"silhouette\" / \"human_reference.json\"\n)\n\n# Flag the document when the composite reaches this penalty (research-validated).\nPENALTY_THRESHOLD = 1.0\n\n# Minimum prose paragraphs before the silhouette metrics are meaningful.\nMIN_PARAGRAPHS = 3\n\n\n# silhouette_scan's stopword list is a pure superset of structure_scan's: same\n# core function words plus pronouns/quantifiers that matter for content-bigram\n# and callback-content comparisons but that structure_scan doesn't need.\nSILHOUETTE_STOPWORDS = _STRUCTURE_STOPWORDS | frozenset({\n \"our\", \"your\", \"their\", \"my\", \"me\", \"us\", \"them\", \"his\", \"her\", \"what\",\n \"which\", \"who\", \"when\", \"where\", \"how\", \"why\", \"there\", \"here\", \"about\",\n \"just\", \"more\", \"most\", \"some\", \"all\", \"also\", \"out\", \"up\", \"one\", \"two\",\n \"get\", \"got\", \"like\", \"much\", \"many\", \"very\", \"every\", \"only\",\n})\n\n# Discourse cue classes for paragraph-opener roles. Copied verbatim from the\n# validated research prototype (scratchpad/research-silhouette/silhouette_probe.py).\nROLE_CUES = {\n \"contrast\": r\"^(on the other hand|on one hand|however|conversely|in contrast|\"\n r\"yet|but |perhaps most|that said|still,)\",\n \"addition\": r\"^(moreover|furthermore|additionally|in addition|also,|\"\n r\"another|second|third|next,|finally,|besides)\",\n \"conclusion\": r\"^(in conclusion|ultimately|overall|in the end|to sum|\"\n r\"in summary|as we|remember|the future)\",\n \"enumeration\": r\"^(first,|firstly|1\\.|step \\d|there are)\",\n \"cause\": r\"^(therefore|thus|consequently|as a result|because of)\",\n}\nROLE_RE = {k: re.compile(v, re.I) for k, v in ROLE_CUES.items()}\n\n\ndef content(text: str) -> list[str]:\n return [w for w in words(text) if len(w) > 3 and w not in SILHOUETTE_STOPWORDS]\n\n\ndef paragraphs(text: str) -> list[str]:\n return _prose_paragraphs(text, strip_bold=True)\n\n\n# ---------------- METRICS (verbatim from the validated prototype) ----------------\n\ndef m_scaffold_opener_share(paras: list[str]):\n \"\"\"Share of body paragraphs opening with a discourse-connective/scaffold cue.\"\"\"\n body = paras[1:] if len(paras) > 1 else paras\n if not body:\n return 0.0\n hits = 0\n for p in body:\n for rx in ROLE_RE.values():\n if rx.search(p):\n hits += 1\n break\n return round(hits / len(body), 3)\n\n\ndef m_role_entropy(paras: list[str]):\n \"\"\"Shannon entropy (bits) over opener role classes incl. 'topic' (none).\n A human never opens paragraphs with cue classes -> entropy 0; a template\n rotates several distinct cue classes -> positive entropy.\"\"\"\n if len(paras) \u003c 3:\n return None\n roles = []\n for p in paras:\n r = \"topic\"\n for name, rx in ROLE_RE.items():\n if rx.search(p):\n r = name\n break\n roles.append(r)\n counts = Counter(roles)\n n = len(roles)\n ent = -sum((c / n) * math.log2(c / n) for c in counts.values())\n return round(ent, 3)\n\n\ndef m_preview_fulfillment(paras: list[str]):\n \"\"\"Share of body paragraphs whose opening content word appears in the\n intro paragraph (outline-following / preview-then-fulfill tell).\"\"\"\n if len(paras) \u003c 4:\n return None\n intro = set(content(paras[0]))\n if not intro:\n return 0.0\n body = paras[1:-1] if len(paras) > 2 else paras[1:]\n hits = tot = 0\n for p in body:\n cs = content(p)\n if not cs:\n continue\n tot += 1\n if cs[0] in intro:\n hits += 1\n return round(hits / tot, 3) if tot else 0.0\n\n\ndef m_callback_content(paras: list[str]):\n \"\"\"Content words introduced in the first third, ABSENT from the middle,\n reappearing in the last third: a recap loop, not a sustained topic.\"\"\"\n n = len(paras)\n if n \u003c 5:\n return None\n third = max(1, n // 3)\n early = set().union(*[set(content(paras[i])) for i in range(third)])\n mid = (set().union(*[set(content(paras[i])) for i in range(third, n - third)])\n if n - 2 * third > 0 else set())\n late = set().union(*[set(content(paras[i])) for i in range(n - third, n)])\n cb = (early & late) - mid\n return round(len(cb) / n, 3)\n\n\ndef m_heading_preview(text: str):\n \"\"\"Share of ## heading head-nouns whose key word also appears in the intro\n paragraph = outline preview-then-fulfill. Measures outline-following, not\n heading presence, so it is retained under --genre docs.\"\"\"\n heads = re.findall(r\"(?m)^\\s{0,3}#{2,3}\\s+(.*)$\", text)\n if len(heads) \u003c 3:\n return None\n paras = paragraphs(text)\n intro = set(content(paras[0])) if paras else set()\n if not intro:\n return 0.0\n hit = 0\n for h in heads:\n hc = set(content(h))\n if hc & intro:\n hit += 1\n return round(hit / len(heads), 3)\n\n\n# Metric registry: name -> (paragraph-based? / text-based?, function).\nPARA_METRICS = {\n \"scaffold_opener_share\": m_scaffold_opener_share,\n \"role_entropy_bits\": m_role_entropy,\n \"preview_fulfillment\": m_preview_fulfillment,\n \"callback_content\": m_callback_content,\n}\nTEXT_METRICS = {\n \"heading_preview\": m_heading_preview,\n}\nMETRIC_ORDER = [\n \"scaffold_opener_share\",\n \"role_entropy_bits\",\n \"heading_preview\",\n \"preview_fulfillment\",\n \"callback_content\",\n]\n\nSUGGESTIONS = {\n \"scaffold_opener_share\":\n \"Open body paragraphs on their own specific claim, not a discourse cue.\",\n \"role_entropy_bits\":\n \"Stop rotating 'However / In addition / Ultimately' scaffold openers.\",\n \"heading_preview\":\n \"Headings restate the intro's outline; let sections carry new ground.\",\n \"preview_fulfillment\":\n \"The body just fulfills an outline previewed in the intro; drop the preview.\",\n \"callback_content\":\n \"The ending loops back to opening vocabulary; end on a concrete final point.\",\n}\n\n\ndef compute_metrics(text: str, paras: list[str]) -> dict:\n row = {}\n for name in METRIC_ORDER:\n if name in PARA_METRICS:\n row[name] = PARA_METRICS[name](paras)\n else:\n row[name] = TEXT_METRICS[name](text)\n return row\n\n\ndef load_reference(path: Path) -> dict:\n data = json.loads(path.read_text())\n return data[\"metrics\"]\n\n\ndef relu(x: float) -> float:\n return x if x > 0 else 0.0\n\n\ndef flag(metric, value, threshold, detail, suggestion) -> dict:\n return {\n \"metric\": metric,\n \"value\": value,\n \"threshold\": threshold,\n \"severity\": \"soft\",\n \"detail\": detail,\n \"suggestion\": suggestion,\n }\n\n\n# Metrics a given --genre suppresses outright. See the docstring in scan()\n# for why only callback_content is suppressed under --genre docs.\nGENRE_SUPPRESSIONS = {\n \"docs\": {\"callback_content\"},\n}\n\n\ndef scan(text: str, reference: dict, genre: str = \"prose\") -> dict:\n # Genre is a passthrough echoed for parity with structure_scan.\n # --genre docs suppresses ONLY callback_content: reference docs, specs, and\n # doctrine conventionally reprise opening themes at the end, which is not\n # the essay recap coda the metric exists to catch (SIL rows pin both\n # directions). heading_preview is deliberately retained under docs because\n # it measures outline-following (a tell even in reference docs), and the\n # metric already clears legitimate academic roadmaps (struct17) on its own.\n # --genre social documents that loose social copy rarely has the\n # >=3-paragraph structure these metrics need.\n paras = paragraphs(text)\n base = {\n \"genre\": genre,\n \"prose_paragraphs\": len(paras),\n }\n if len(paras) \u003c MIN_PARAGRAPHS:\n base.update({\n \"flags\": [],\n \"flagged\": {},\n \"metrics\": None,\n \"penalty\": None,\n \"note\": f\"fewer than {MIN_PARAGRAPHS} prose paragraphs; \"\n \"silhouette metrics not scored\",\n })\n return base\n\n metrics = compute_metrics(text, paras)\n contributions = {}\n penalty = 0.0\n flags = []\n for name in METRIC_ORDER:\n if name in GENRE_SUPPRESSIONS.get(genre, set()):\n contributions[name] = 0.0\n continue\n ref = reference[name]\n value = metrics[name]\n if not isinstance(value, (int, float)):\n contributions[name] = None\n continue\n median = ref[\"median\"]\n scale = max(ref[\"iqr\"], ref[\"fence\"])\n weight = ref[\"weight\"]\n contribution = round(weight * relu((value - median) / scale), 3)\n contributions[name] = contribution\n penalty += contribution\n # A metric individually clears the human fence when value >= fence,\n # i.e. its contribution reaches its full weight.\n if value >= ref[\"fence\"]:\n flags.append(flag(\n name,\n value,\n f\"human fence {ref['fence']} (weight {weight})\",\n f\"{name} at {value} clears the human upper fence \"\n f\"{ref['fence']}.\",\n SUGGESTIONS[name],\n ))\n penalty = round(penalty, 3)\n\n if penalty >= PENALTY_THRESHOLD:\n flags.insert(0, flag(\n \"silhouette_penalty\",\n penalty,\n f\">= {PENALTY_THRESHOLD}\",\n \"The document's idea arrangement matches a templated AI silhouette \"\n \"(preview-then-fulfill, rotating scaffold openers, recap loop).\",\n \"Rearrange around the actual argument instead of a symmetric outline; \"\n \"cut previews and the closing recap.\",\n ))\n\n base.update({\n \"flags\": flags,\n \"flagged\": {f[\"metric\"]: True for f in flags},\n \"metrics\": metrics,\n \"contributions\": contributions,\n \"penalty\": penalty,\n })\n return base\n\n\ndef parse_args(argv: list[str]) -> argparse.Namespace:\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\"path\", nargs=\"?\")\n parser.add_argument(\"--genre\", choices=[\"prose\", \"docs\", \"social\"], default=\"prose\")\n return parser.parse_args(argv)\n\n\ndef main(argv: list[str]) -> int:\n args = parse_args(argv)\n if not REFERENCE_PATH.exists():\n print(f\"Missing reference: {REFERENCE_PATH}\", file=sys.stderr)\n return 2\n reference = load_reference(REFERENCE_PATH)\n\n if args.path:\n path = Path(args.path)\n if not path.exists():\n print(f\"Missing file: {path}\", file=sys.stderr)\n return 2\n text = path.read_text(errors=\"replace\")\n else:\n text = sys.stdin.buffer.read().decode(\"utf-8\", errors=\"replace\")\n\n result = scan(text, reference, args.genre)\n\n if not result.get(\"flags\") and not is_probably_english(text):\n print(json.dumps({\"non_english\": True, \"flags\": [], \"penalty\": None}, indent=2))\n print(\"note: input appears non-English; scanner declined (English-only).\", file=sys.stderr)\n return 0\n\n print(json.dumps(result, indent=2))\n is_flagged = bool(result.get(\"penalty\") is not None\n and result[\"penalty\"] >= PENALTY_THRESHOLD)\n return 1 if is_flagged else 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main(sys.argv[1:]))\n","content_type":"text/x-python; charset=utf-8","language":"python","size":14242,"content_sha256":"431f08dc52a705370d48bc2df72d10a87e5afb91a2b8266c0a2aa1d1b5fdda5c"},{"filename":"scripts/structure_scan.py","content":"#!/usr/bin/env python3\n\"\"\"Scan prose for macro-structure AI-writing patterns.\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport math\nimport re\nimport sys\nfrom collections import Counter\nfrom pathlib import Path\n\nHERE = Path(__file__).resolve().parent\nsys.path.insert(0, str(HERE))\n\nfrom _lang import ( # noqa: E402\n ENGLISH_FUNCTION_WORDS,\n english_function_share,\n is_probably_english,\n paragraphs as _prose_paragraphs,\n words,\n)\nfrom readability_metrics import split_sentences # noqa: E402\n\n\nSTOPWORDS = {\n \"the\", \"a\", \"an\", \"and\", \"or\", \"but\", \"in\", \"on\", \"at\", \"to\", \"for\", \"of\",\n \"with\", \"by\", \"from\", \"is\", \"are\", \"was\", \"were\", \"be\", \"been\", \"being\",\n \"have\", \"has\", \"had\", \"do\", \"does\", \"did\", \"will\", \"would\", \"could\",\n \"should\", \"may\", \"might\", \"must\", \"can\", \"it\", \"its\", \"this\", \"that\",\n \"these\", \"those\", \"we\", \"you\", \"they\", \"i\", \"he\", \"she\", \"as\", \"than\",\n \"if\", \"then\", \"so\", \"not\", \"no\", \"yes\", \"into\", \"over\", \"under\",\n}\n\n\nCONNECTIVE_OPENERS = re.compile(\n r\"^(however|moreover|furthermore|additionally|in addition|overall|\"\n r\"consequently|nevertheless)\\b\",\n re.I,\n)\n# Paragraph-initial \"Every \u003cnoun> \u003cverb-phrase>\" template opener (\"Every\n# transformation is scored ...\", \"Every rewrite passes ...\"). As a RHYTHM tell\n# this is about REPETITION, so the metric below fires only at 2+ such paragraph\n# openers; a lone \"Every child deserves a good school.\" stays clean. The verb is\n# a copula/auxiliary or a present/past inflection (-s / -ed) so the second word\n# reads as a predicate, not another noun.\nEVERY_OPENER_RE = re.compile(\n r\"^every\\s+[a-z][\\w'-]*\\s+(?:is|are|was|were|be|been|being|has|have|had|\"\n r\"do|does|did|can|will|shall|must|should|would|may|might|\"\n r\"[a-z]+(?:s|ed))\\b\",\n re.I,\n)\nSIGNPOST_RE = re.compile(\n r\"\\b(first,|next,|having (?:covered|established)|in (?:this|the following|\"\n r\"the next) section|as (?:mentioned|noted) (?:above|earlier)|let us turn|\"\n r\"let's turn)\\b\",\n re.I,\n)\nCLOSER_RE = re.compile(\n r\",\\s+(ensuring|highlighting|reflecting|allowing|enabling|underscoring|\"\n r\"showcasing|emphasizing|fostering|driving|paving|reinforcing|solidifying|\"\n r\"demonstrating|contributing to)\\b[^.!?]*[.!?\\\"]?$\",\n re.I,\n)\nCODA_START_RE = re.compile(\n r\"^(ultimately,|in the end,|in conclusion|as we've seen|only time will tell|\"\n r\"remember,|the future\\b)\",\n re.I,\n)\nBOLD_COLON_RE = re.compile(r\"^\\s*[-*+]?\\s*\\*\\*[^*]{1,40}\\*\\*\\s*:\", re.M)\n\n\ndef prose_paragraphs(text: str) -> list[str]:\n return _prose_paragraphs(text, blank_blockquotes=True)\n\n\ndef cv(values: list[int]) -> float:\n if not values:\n return 0.0\n mean = sum(values) / len(values)\n if mean == 0:\n return 0.0\n return math.sqrt(sum((v - mean) ** 2 for v in values) / len(values)) / mean\n\n\ndef content_bigrams(text: str) -> set[tuple[str, str]]:\n toks = [w for w in words(text) if len(w) > 3 and w not in STOPWORDS]\n return set(zip(toks, toks[1:]))\n\n\ndef triad_count(text: str) -> int:\n return len(re.findall(r\"\\b[A-Za-z][A-Za-z-]+,\\s+[A-Za-z][A-Za-z-]+,\\s+and\\s+[A-Za-z][A-Za-z-]+\\b\", text))\n\n\ndef flag(metric: str, value: float | int, threshold: str, detail: str, suggestion: str) -> dict:\n return {\n \"metric\": metric,\n \"value\": value,\n \"threshold\": threshold,\n \"severity\": \"soft\",\n \"detail\": detail,\n \"suggestion\": suggestion,\n }\n\n\n# Metrics a given --genre suppresses outright (the genre's normal shape trips\n# the metric without being AI-slop). Table form so each suppression is a single\n# lookup instead of an inline `genre != \"...\"` conditional per metric.\nGENRE_SUPPRESSIONS = {\n \"docs\": {\"bold_colon_listicle\"},\n \"social\": {\"one_line_staccato\"},\n}\n\n\ndef scan(text: str, genre: str = \"prose\") -> dict:\n paragraphs = prose_paragraphs(text)\n prose_text = \"\\n\\n\".join(paragraphs)\n sentences = split_sentences(prose_text)\n sentence_lengths = [len(words(s)) for s in sentences]\n prose_words = words(prose_text)\n para_lengths = [len(words(p)) for p in paragraphs]\n metrics = {\n \"sentence_burstiness\": round(cv(sentence_lengths), 3),\n \"summary_sandwich\": 0.0,\n \"paragraph_cv\": round(cv(para_lengths), 3),\n \"sentence_mean_len\": round(sum(sentence_lengths) / len(sentence_lengths), 1) if sentence_lengths else 0,\n \"triad_density\": round((triad_count(prose_text) / len(prose_words) * 1000), 3) if prose_words else 0,\n \"em_dash_per_1k\": round((text.count(\"—\") / len(prose_words) * 1000), 3) if prose_words else 0,\n \"bold_colon_listicle_count\": len(BOLD_COLON_RE.findall(text)),\n \"one_line_staccato_share\": 0.0,\n \"connective_paragraph_openers\": 0,\n \"every_template_openers\": 0,\n \"signpost_density\": 0.0,\n \"opener_unique_ratio\": 0.0,\n \"top_opener_share\": 0.0,\n \"max_consecutive_opener\": 0,\n \"participial_closer_share\": 0.0,\n \"conclusion_coda\": False,\n }\n flags = []\n\n if len(paragraphs) >= 2:\n first = content_bigrams(paragraphs[0])\n last = content_bigrams(paragraphs[-1])\n union = first | last\n metrics[\"summary_sandwich\"] = round(len(first & last) / len(union), 3) if union else 0.0\n\n if len(sentences) >= 8 and metrics[\"sentence_burstiness\"] \u003c 0.55:\n flags.append(flag(\n \"sentence_burstiness\",\n metrics[\"sentence_burstiness\"],\n \"\u003c 0.55 over at least 8 prose sentences\",\n \"Sentence lengths are unusually uniform for running prose.\",\n \"Vary sentence length and cadence; if this is formal reference prose, review before treating it as blocking.\",\n ))\n\n if len(paragraphs) >= 3:\n coda = bool(CODA_START_RE.search(paragraphs[-1]))\n if not coda:\n coda = len(content_bigrams(paragraphs[0]) & content_bigrams(paragraphs[-1])) >= 2\n metrics[\"conclusion_coda\"] = coda\n if coda:\n flags.append(flag(\n \"conclusion_coda\",\n 1,\n \"last paragraph starts with a stock coda or repeats 2+ first-paragraph content bigrams\",\n \"The ending reads like a recap/moral coda instead of a concrete final point.\",\n \"Cut the wrap-up or end on a specific fact; if this is an abstract or executive summary, judge the genre before changing it.\",\n ))\n\n if (metrics[\"bold_colon_listicle_count\"] >= 3\n and \"bold_colon_listicle\" not in GENRE_SUPPRESSIONS.get(genre, set())):\n flags.append(flag(\n \"bold_colon_listicle\",\n metrics[\"bold_colon_listicle_count\"],\n \">= 3 bold-label colon lines\",\n \"The raw Markdown has repeated bold-label listicle formatting.\",\n \"Convert to prose or plain bullets; if this is a reference doc, rerun with --genre docs.\",\n ))\n\n if paragraphs:\n one_line = 0\n for p in paragraphs:\n ps = split_sentences(p)\n if len(ps) == 1 and len(words(ps[0])) \u003c 12:\n one_line += 1\n metrics[\"one_line_staccato_share\"] = round(one_line / len(paragraphs), 3)\n if (len(paragraphs) >= 6 and metrics[\"one_line_staccato_share\"] > 0.6\n and \"one_line_staccato\" not in GENRE_SUPPRESSIONS.get(genre, set())):\n flags.append(flag(\n \"one_line_staccato\",\n metrics[\"one_line_staccato_share\"],\n \"> 0.60 over at least 6 paragraphs\",\n \"Most paragraphs are short single-sentence beats.\",\n \"Merge related beats and vary paragraph length; if this is social copy, rerun with --genre social.\",\n ))\n\n connective = sum(1 for p in paragraphs if CONNECTIVE_OPENERS.search(p))\n metrics[\"connective_paragraph_openers\"] = connective\n if connective >= 3 or (len(paragraphs) >= 8 and connective / len(paragraphs) > 0.4):\n flags.append(flag(\n \"connective_paragraph_openers\",\n connective,\n \">= 3 paragraphs or > 40% of 8+ paragraphs\",\n \"Paragraphs repeatedly open with formal transition words.\",\n \"Replace scaffold openers with specific topic sentences; academic prose may justify some connectors.\",\n ))\n\n every_openers = sum(1 for p in paragraphs if EVERY_OPENER_RE.search(p))\n metrics[\"every_template_openers\"] = every_openers\n if every_openers >= 2:\n flags.append(flag(\n \"every_template_openers\",\n every_openers,\n \">= 2 paragraphs opening 'Every \u003cnoun> \u003cverb>'\",\n \"Paragraphs repeatedly open on the 'Every ___ is/does ...' template.\",\n \"Vary the paragraph openings; a repeated Every-template is a machine rhythm tell even when each sentence is fine on its own.\",\n ))\n\n if prose_words:\n signposts = len(SIGNPOST_RE.findall(prose_text))\n metrics[\"signpost_density\"] = round(signposts / len(prose_words) * 100, 3)\n if len(prose_words) >= 150 and metrics[\"signpost_density\"] > 0.6:\n flags.append(flag(\n \"signpost_density\",\n metrics[\"signpost_density\"],\n \"> 0.6 per 100 prose words, minimum 150 words\",\n \"The text over-explains its own structure.\",\n \"Remove roadmap language unless the genre is a textbook, legal brief, or long guide.\",\n ))\n\n if len(sentences) >= 5:\n openers = []\n for s in sentences:\n ws = words(s)\n if not ws:\n continue\n openers.append(ws[0])\n enumeration = {\"the\", \"a\", \"an\", \"section\", \"chapter\", \"figure\", \"table\",\n \"step\", \"part\", \"appendix\"}\n counted = [o for o in openers if o not in enumeration]\n top_count = 0\n if counted:\n counts = Counter(counted)\n top_count = max(counts.values())\n metrics[\"opener_unique_ratio\"] = round(len(counts) / len(counted), 3)\n metrics[\"top_opener_share\"] = round(top_count / len(counted), 3)\n run = max_run = 0\n prev = None\n for opener in counted:\n run = run + 1 if opener == prev else 1\n prev = opener\n max_run = max(max_run, run)\n metrics[\"max_consecutive_opener\"] = max_run\n top_repeat = metrics[\"top_opener_share\"] > 0.25 and top_count >= 4\n if metrics[\"opener_unique_ratio\"] \u003c 0.55 or top_repeat or max_run >= 4:\n flags.append(flag(\n \"opener_repetition\",\n metrics[\"opener_unique_ratio\"],\n \"unique ratio \u003c 0.55, one opener > 25% with 4+ uses, or 4 consecutive identical openers\",\n \"Sentence openings repeat in a template-like rhythm.\",\n \"Rewrite repeated starts; step-by-step docs may need repeated imperative openers.\",\n ))\n\n if sentences:\n closer_count = sum(1 for s in sentences if CLOSER_RE.search(s))\n metrics[\"participial_closer_share\"] = round(closer_count / len(sentences), 3)\n if len(sentences) >= 8 and metrics[\"participial_closer_share\"] >= 0.15:\n flags.append(flag(\n \"participial_closer_share\",\n metrics[\"participial_closer_share\"],\n \">= 0.15 over at least 8 prose sentences\",\n \"Many sentences end with editorial -ing consequence tails.\",\n \"Make the consequence concrete or cut the tail; analytical prose may allow an occasional closer.\",\n ))\n\n return {\n \"flags\": flags,\n \"flagged\": {f[\"metric\"]: True for f in flags},\n \"metrics\": metrics,\n \"genre\": genre,\n \"prose_sentences\": len(sentences),\n \"prose_paragraphs\": len(paragraphs),\n }\n\n\ndef parse_args(argv: list[str]) -> argparse.Namespace:\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\"path\", nargs=\"?\")\n parser.add_argument(\"--genre\", choices=[\"prose\", \"docs\", \"social\"], default=\"prose\")\n return parser.parse_args(argv)\n\n\ndef main(argv: list[str]) -> int:\n args = parse_args(argv)\n if args.path:\n path = Path(args.path)\n if not path.exists():\n print(f\"Missing file: {path}\", file=sys.stderr)\n return 2\n text = path.read_text(errors=\"replace\")\n else:\n text = sys.stdin.buffer.read().decode(\"utf-8\", errors=\"replace\")\n\n # English-only graceful decline, matching banned_phrase_scan.py.\n result = scan(text, args.genre)\n\n # Function-word absence alone is not evidence of a foreign language\n # (imperative stacks and buzzword lists are English slop with few function\n # words). Decline only when the heuristic fails AND nothing flagged.\n if not result.get(\"flags\") and not is_probably_english(text):\n print(json.dumps({\"non_english\": True, \"violations\": [], \"flags\": []}, indent=2))\n print(\"note: input appears non-English; scanner declined (English-only).\", file=sys.stderr)\n return 0\n\n print(json.dumps(result, indent=2))\n return 1 if result[\"flags\"] else 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main(sys.argv[1:]))\n","content_type":"text/x-python; charset=utf-8","language":"python","size":13157,"content_sha256":"a8ff628a85ac3fd601cb0caf297ad532722d54e7cdd64623357fa626bd872beb"},{"filename":"scripts/suggest.py","content":"#!/usr/bin/env python3\n\"\"\"Co-writer suggestion mode: emit LSP-style structured edit suggestions.\n\nDetection is cheap and deterministic (banned_phrase_scan + structure_scan).\nReplacement generation is DELEGATED to a stronger model: by default every\nsuggestion is emitted with ``\"suggested_replacement\": null``. A separate\n``--apply-replacements FILE`` mode merges externally-produced replacements back\nin and light-validates them. The blocking contract gates live in\ncheck_suggestions.py; this script never rewrites the document itself.\n\nOutput shape:\n {\n \"document\": \"\u003coriginal text>\",\n \"suggestions\": [\n {\n \"span\": {\"start\": N, \"end\": N, \"text\": \"...\"},\n \"severity\": \"hard\" | \"soft\",\n \"category\": \"...\",\n \"rationale\": \"...\",\n \"suggested_replacement\": null,\n \"phrased_as_question\": bool\n },\n ...\n ],\n \"counts\": {...}\n }\n\nSoft findings are phrased as questions (register-dependent judgment calls);\nhard findings are stated as direct replacements. Suggestions are emitted in a\ndeterministic order (span start, then end, then category) and never overlap.\n\nUsage:\n python3 scripts/suggest.py document.md\n python3 scripts/suggest.py \u003c document.md\n python3 scripts/suggest.py document.md --apply-replacements replacements.json\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport re\nimport sys\nfrom pathlib import Path\n\nfrom banned_phrase_scan import (\n is_probably_english,\n scan_for_violations,\n)\nfrom structure_scan import scan as structure_scan\n\n\ndef _line_starts(text: str) -> list[int]:\n \"\"\"Character offset at which each line begins (index 0 == line 1).\"\"\"\n starts = [0]\n for i, ch in enumerate(text):\n if ch == \"\\n\":\n starts.append(i + 1)\n return starts\n\n\ndef _offset(line_starts: list[int], line_number: int, column: int) -> int:\n return line_starts[line_number - 1] + (column - 1)\n\n\ndef _rationale(category: str, span_text: str, suggestion: str | None, is_soft: bool) -> str:\n \"\"\"Build a reviewer-facing rationale.\n\n Soft findings are worded as a question because they are judgment calls whose\n right answer depends on register; hard findings are stated as a direct fix.\n \"\"\"\n if is_soft:\n hint = f\" Consider: {suggestion}.\" if suggestion else \"\"\n return f\"Soft tell ({category}): could “{span_text}” be cut or reworded here?{hint}\"\n if suggestion:\n return f\"AI-writing tell ({category}): replace “{span_text}” — {suggestion}\"\n return f\"AI-writing tell ({category}): replace “{span_text}”.\"\n\n\ndef build_suggestions(text: str) -> list[dict]:\n \"\"\"Detect AI-isms and turn each into a span-anchored suggestion.\n\n Spans come from banned_phrase_scan (phrases + structural regexes), which are\n the findings with real character offsets that a replacement can be applied\n to. macro-structure flags from structure_scan are document-level and are\n surfaced in ``counts`` instead, since they have no single applyable span.\n \"\"\"\n violations = scan_for_violations(text)\n line_starts = _line_starts(text)\n candidates: list[dict] = []\n for v in violations:\n start = _offset(line_starts, v[\"line_number\"], v[\"column\"])\n end = start + len(v[\"phrase\"])\n span_text = text[start:end]\n is_soft = v[\"severity\"] == \"soft\"\n candidates.append({\n \"span\": {\"start\": start, \"end\": end, \"text\": span_text},\n \"severity\": v[\"severity\"],\n \"category\": v[\"category\"],\n \"rationale\": _rationale(v[\"category\"], span_text, v.get(\"suggestion\"), is_soft),\n \"suggested_replacement\": None,\n \"phrased_as_question\": is_soft,\n })\n\n candidates.sort(key=lambda s: (s[\"span\"][\"start\"], s[\"span\"][\"end\"], s[\"category\"]))\n\n # Enforce non-overlap deterministically: keep the earliest-starting span and\n # drop any later suggestion that overlaps an already-kept one, so the emitted\n # set always satisfies the span-overlap contract gate.\n kept: list[dict] = []\n last_end = -1\n for s in candidates:\n if s[\"span\"][\"start\"] >= last_end:\n kept.append(s)\n last_end = s[\"span\"][\"end\"]\n return kept\n\n\ndef counts_block(suggestions: list[dict], struct: dict) -> dict:\n by_category: dict[str, int] = {}\n hard = soft = 0\n for s in suggestions:\n by_category[s[\"category\"]] = by_category.get(s[\"category\"], 0) + 1\n if s[\"severity\"] == \"soft\":\n soft += 1\n else:\n hard += 1\n return {\n \"total\": len(suggestions),\n \"hard\": hard,\n \"soft\": soft,\n \"by_category\": by_category,\n \"structure_flags\": [f[\"metric\"] for f in struct.get(\"flags\", [])],\n }\n\n\ndef apply_replacements(suggestions: list[dict], repl_path: str) -> list[str]:\n \"\"\"Merge externally-produced replacements into suggestions and light-validate.\n\n The replacement file is ``{\"replacements\": [{\"start\", \"end\", \"replacement\"}]}``.\n Each replacement is keyed to a suggestion by exact (start, end) span. This is\n only a light merge/validation pass; the blocking contract lives in\n check_suggestions.py.\n \"\"\"\n data = json.loads(Path(repl_path).read_text())\n index = {(r[\"start\"], r[\"end\"]): r[\"replacement\"] for r in data.get(\"replacements\", [])}\n warnings: list[str] = []\n matched: set[tuple[int, int]] = set()\n for s in suggestions:\n key = (s[\"span\"][\"start\"], s[\"span\"][\"end\"])\n if key not in index:\n continue\n rep = index[key]\n s[\"suggested_replacement\"] = rep\n matched.add(key)\n if rep == s[\"span\"][\"text\"]:\n warnings.append(f\"replacement for span {list(key)} is identical to span text\")\n elif scan_for_violations(rep) or structure_scan(rep).get(\"flags\"):\n warnings.append(f\"replacement for span {list(key)} does not pass the scanners in isolation\")\n for key in index:\n if key not in matched:\n warnings.append(f\"replacement targets unknown span {list(key)}\")\n return warnings\n\n\ndef parse_args(argv: list[str]) -> argparse.Namespace:\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\"path\", nargs=\"?\", help=\"Document file. Reads stdin when omitted.\")\n parser.add_argument(\n \"--apply-replacements\",\n metavar=\"FILE\",\n help=\"Merge externally-produced replacements (JSON) into the suggestions.\",\n )\n return parser.parse_args(argv)\n\n\ndef main(argv: list[str]) -> int:\n args = parse_args(argv)\n if args.path:\n path = Path(args.path)\n if not path.exists():\n print(f\"Missing file: {path}\", file=sys.stderr)\n return 2\n text = path.read_text(errors=\"replace\")\n else:\n text = sys.stdin.buffer.read().decode(\"utf-8\", errors=\"replace\")\n\n # English-only graceful decline, matching the two scanners.\n if not is_probably_english(text):\n print(json.dumps({\"non_english\": True, \"document\": text, \"suggestions\": [],\n \"counts\": {\"total\": 0, \"hard\": 0, \"soft\": 0,\n \"by_category\": {}, \"structure_flags\": []}},\n indent=2))\n print(\"note: input appears non-English; co-writer declined (English-only).\", file=sys.stderr)\n return 0\n\n suggestions = build_suggestions(text)\n struct = structure_scan(text)\n out = {\n \"document\": text,\n \"suggestions\": suggestions,\n \"counts\": counts_block(suggestions, struct),\n }\n if args.apply_replacements:\n out[\"apply_warnings\"] = apply_replacements(suggestions, args.apply_replacements)\n print(json.dumps(out, indent=2))\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main(sys.argv[1:]))\n","content_type":"text/x-python; charset=utf-8","language":"python","size":7832,"content_sha256":"1d93bb9444ae4b5218fb81daf1b41ff9f09b75c950e3fb8344c582bf7eed4050"},{"filename":"scripts/validate_preservation.py","content":"#!/usr/bin/env python3\n\"\"\"\nValidate that all must-preserve constraints survived transformation.\n\nCompares original text constraints against transformed text.\nExit code 0 = all constraints preserved, 1 = missing constraints.\n\nUsage:\n python validate_preservation.py original.txt transformed.txt\n python validate_preservation.py original.txt transformed.txt constraints.json\n\"\"\"\n\nfrom __future__ import annotations\n\nimport sys\nimport json\nimport re\nfrom typing import TypedDict\n\n# Import constraint extraction\nfrom extract_constraints import extract_constraints, Constraint\n\n\nclass ValidationResult(TypedDict):\n passed: bool\n total_constraints: int\n preserved: int\n missing: list[Constraint]\n warnings: list[str]\n\n\n_MAGNITUDES = {\n \"k\": 1e3, \"thousand\": 1e3,\n \"m\": 1e6, \"million\": 1e6,\n \"b\": 1e9, \"billion\": 1e9,\n \"trillion\": 1e12,\n}\n\n_MONTHS = [\n \"january\", \"february\", \"march\", \"april\", \"may\", \"june\", \"july\",\n \"august\", \"september\", \"october\", \"november\", \"december\",\n]\n\n\ndef normalize_value(value: str) -> str:\n \"\"\"Normalize a constraint value for comparison.\"\"\"\n # Remove extra whitespace\n normalized = re.sub(r'\\s+', ' ', value.strip())\n # Lowercase for comparison (preserve original for display)\n return normalized.lower()\n\n\ndef parse_money(token: str) -> float | None:\n \"\"\"Parse a currency token into an absolute amount, honoring magnitude words.\n\n '$47.3M' and '$47.3 million' -> 47_300_000; '$47.3 billion' -> 47_300_000_000.\n Magnitude is part of the fact, so the two must not compare equal.\n \"\"\"\n m = re.search(\n r'([\\d,]+\\.?\\d*)\\s*(k|m|b|thousand|million|billion)?',\n token.lower().replace(\"$\", \"\"),\n )\n if not m or not m.group(1).strip(\",\"):\n return None\n amount = float(m.group(1).replace(\",\", \"\"))\n if m.group(2):\n amount *= _MAGNITUDES[m.group(2)]\n return amount\n\n\ndef parse_magnitude_number(token: str) -> float | None:\n m = re.search(\n r'\\b([\\d,]+\\.?\\d*)\\s*(thousand|million|billion|trillion)?\\b',\n token.lower(),\n )\n if not m or not m.group(1).strip(\",\"):\n return None\n amount = float(m.group(1).replace(\",\", \"\"))\n if m.group(2):\n amount *= _MAGNITUDES[m.group(2)]\n return amount\n\n\ndef _numbers_match_exactly(value: str, text: str, pattern: str) -> bool:\n \"\"\"True iff the numeric value appears as a whole quantity in text.\n\n Guards against the substring trap where '12' is 'found' inside '120'.\n \"\"\"\n want = re.findall(r'\\d+\\.?\\d*', value)\n have = set(re.findall(pattern, text.lower()))\n return all(num in have for num in want) and bool(want)\n\n\n_UNIT_SYNONYMS = {\n \"km\": {\"km\", \"kilometer\", \"kilometers\", \"kilometre\", \"kilometres\"},\n \"mi\": {\"mi\", \"mile\", \"miles\"},\n \"kg\": {\"kg\", \"kilogram\", \"kilograms\"},\n \"g\": {\"g\", \"gram\", \"grams\"},\n \"ms\": {\"ms\", \"millisecond\", \"milliseconds\"},\n \"s\": {\"s\", \"sec\", \"second\", \"seconds\"},\n \"m\": {\"m\", \"meter\", \"meters\", \"metre\", \"metres\"},\n \"cm\": {\"cm\", \"centimeter\", \"centimeters\", \"centimetre\", \"centimetres\"},\n \"mm\": {\"mm\", \"millimeter\", \"millimeters\", \"millimetre\", \"millimetres\"},\n \"ft\": {\"ft\", \"foot\", \"feet\"},\n \"in\": {\"in\", \"inch\", \"inches\"},\n \"lb\": {\"lb\", \"lbs\", \"pound\", \"pounds\"},\n \"oz\": {\"oz\", \"ounce\", \"ounces\"},\n \"°c\": {\"°c\", \"c\", \"celsius\"},\n \"°f\": {\"°f\", \"f\", \"fahrenheit\"},\n \"min\": {\"min\", \"mins\", \"minute\", \"minutes\"},\n \"hr\": {\"hr\", \"hrs\", \"hour\", \"hours\"},\n \"day\": {\"day\", \"days\"},\n \"week\": {\"week\", \"weeks\", \"wk\", \"wks\"},\n \"month\": {\"month\", \"months\", \"mo\"},\n \"year\": {\"year\", \"years\", \"yr\", \"yrs\"},\n \"kb\": {\"kb\", \"kilobyte\", \"kilobytes\"},\n \"mb\": {\"mb\", \"megabyte\", \"megabytes\"},\n \"gb\": {\"gb\", \"gigabyte\", \"gigabytes\"},\n \"tb\": {\"tb\", \"terabyte\", \"terabytes\"},\n \"pb\": {\"pb\", \"petabyte\", \"petabytes\"},\n \"px\": {\"px\", \"pixel\", \"pixels\"},\n}\n\n\ndef _measurement_parts(value: str) -> tuple[str, set[str]] | None:\n m = re.search(r'(\\d+\\.?\\d*)\\s*([^\\d\\s]+|degrees?(?:\\s+\\w+)?)', value, re.I)\n if not m:\n return None\n unit = m.group(2).lower().strip()\n unit = re.sub(r'^degrees?\\s*', '', unit) or \"degree\"\n aliases = _UNIT_SYNONYMS.get(unit)\n if aliases is None:\n for family in _UNIT_SYNONYMS.values():\n if unit in family:\n aliases = family\n break\n else:\n aliases = {unit}\n return m.group(1), aliases\n\n\ndef _quote_core(value: str) -> str:\n return value.strip().strip('\"“”').lower()\n\n\ndef _time_variants(value: str) -> set[str]:\n m = re.search(r'\\b(\\d{1,2})(?::(\\d{2}))?(?::\\d{2})?\\s*(am|pm)?\\b', value, re.I)\n if not m:\n return set()\n hour = str(int(m.group(1)))\n minute = m.group(2) or \"00\"\n suffix = (m.group(3) or \"\").lower()\n spaced = f\" {suffix}\" if suffix else \"\"\n compact = suffix\n variants = {f\"{hour}:{minute}{spaced}\".strip(), f\"{hour}:{minute}{compact}\".strip()}\n if minute == \"00\":\n variants.update({f\"{hour}{spaced}\".strip(), f\"{hour}{compact}\".strip()})\n return variants\n\n\ndef find_constraint_in_text(constraint: Constraint, text: str) -> bool:\n \"\"\"Check if a constraint value exists in text.\"\"\"\n value = constraint[\"value\"]\n ctype = constraint[\"type\"]\n normalized_value = normalize_value(value)\n normalized_text = normalize_value(text)\n\n # Direct match\n if normalized_value in normalized_text:\n return True\n\n # and/or: the disjunction must survive; \"or\" (or \"or both\") is faithful,\n # a bare conjunction is not.\n if ctype == \"and_or\":\n return bool(re.search(r\"\\band/or\\b|\\bor\\b\", text, re.IGNORECASE))\n\n # Currency: compare absolute amounts so a magnitude swap (M -> billion) fails.\n if ctype == \"currency\":\n target = parse_money(value)\n if target is None:\n return False\n for token in re.findall(\n r'\\$?[\\d,]+\\.?\\d*\\s*(?:k|m|b|thousand|million|billion)?', text, re.IGNORECASE\n ):\n amount = parse_money(token)\n if amount is not None and abs(amount - target) \u003c 0.01:\n return True\n return False\n\n if ctype == \"magnitude_number\":\n target = parse_magnitude_number(value)\n if target is None:\n return False\n for token in re.findall(\n r'\\b[\\d,]+\\.?\\d*\\s*(?:thousand|million|billion|trillion)?\\b',\n text,\n re.IGNORECASE,\n ):\n amount = parse_magnitude_number(token)\n if amount is not None and abs(amount - target) \u003c 0.01:\n return True\n return False\n\n # Percentage: require an exact numeric match (12% != 120%).\n if ctype == \"percentage\":\n return _numbers_match_exactly(value, text, r'(\\d+\\.?\\d*)\\s*(?:%|percent)')\n\n if ctype == \"measurement\":\n parts = _measurement_parts(value)\n if not parts:\n return False\n number, units = parts\n clean_text = text.replace(\",\", \"\").lower()\n if not re.search(r'(?\u003c![\\d.])' + re.escape(number.replace(\",\", \"\")) + r'(?![\\d.])', clean_text):\n return False\n return any(re.search(r'(?\u003c!\\w)' + re.escape(unit) + r'(?!\\w)', clean_text) for unit in units)\n\n if ctype == \"range\":\n return _numbers_match_exactly(value, text, r'(?\u003c![\\d.])(\\d+\\.?\\d*)(?![\\d.])')\n\n if ctype == \"time\":\n text_variants = _time_variants(text)\n return bool(_time_variants(value) & text_variants)\n\n # Other quantities: match digits, comma-insensitive, but as whole tokens.\n if ctype in (\"count\", \"number\"):\n for num in re.findall(r'[\\d,]+\\.?\\d*', value):\n clean_num = num.replace(\",\", \"\")\n if re.search(r'(?\u003c![\\d.])' + re.escape(clean_num) + r'(?![\\d.])',\n text.replace(\",\", \"\")):\n return True\n return False\n\n # Dates: the year alone is not enough — a changed month is a changed fact.\n if ctype == \"date_quarter\":\n year_match = re.search(r'\\d{4}', value)\n if not (year_match and year_match.group() in text):\n return False\n q = re.search(r'q([1-4])', value.lower())\n if not q:\n return False\n ordinal = {\"1\": \"first\", \"2\": \"second\", \"3\": \"third\", \"4\": \"fourth\"}[q.group(1)]\n low_text = text.lower()\n return q.group(0) in low_text or re.search(ordinal + r'\\s+quarter', low_text) is not None\n\n if ctype.startswith(\"date\"):\n year_match = re.search(r'\\d{4}', value)\n if not (year_match and year_match.group() in text):\n return False\n low = value.lower()\n month = next((mo for mo in _MONTHS if mo[:3] in low), None)\n if month and month[:3] not in text.lower():\n return False\n quarter = re.search(r'q[1-4]', low)\n if quarter and quarter.group() not in text.lower():\n return False\n return True\n\n # For quotes, check if core content is present (without surrounding quotes)\n if constraint[\"type\"] == \"quote\":\n inner = _quote_core(value)\n comparable_text = normalized_text.replace(\"“\", '\"').replace(\"”\", '\"')\n if inner in comparable_text:\n return True\n\n if ctype == \"proper_noun\":\n words = re.findall(r'[A-Z][a-z]+', value)\n if words and all(re.search(r'\\b' + re.escape(word) + r'\\b', text, re.I) for word in words):\n return True\n\n return False\n\n\n_NEGATION_RE = re.compile(\n r\"\\b(?:not|never|no|cannot|can't|won't|don't|doesn't|didn't|isn't|aren't|\"\n r\"wasn't|weren't|without|neither|nor|none|fails?\\s+to|rather\\s+than)\\b\", re.I)\n_SCOPE_RE = re.compile(\n r\"\\b(?:most|all|none|every|each|some|few|several|majority|minority|only|\"\n r\"always|usually|typically|rarely|approximately|roughly|about|nearly)\\b\", re.I)\n_CONDITIONAL_RE = re.compile(\n r\"\\b(?:if|unless|provided\\s+that|only\\s+if|assuming|given\\s+that|\"\n r\"in\\s+the\\s+event|contingent\\s+on|except|excluding|other\\s+than|\"\n r\"aside\\s+from|save\\s+for)\\b\", re.I)\n_WEAK_MODAL_RE = re.compile(\n r\"\\b(?:may|might|could|should|can|likely|probably|appears?|suggests?|seems?)\\b\", re.I)\n_STRONG_MODAL_RE = re.compile(\n r\"\\b(?:will|must|always|definitely|certainly|guarantees?|proves?)\\b\", re.I)\n\n\ndef semantic_drift_warnings(original: str, transformed: str) -> list[str]:\n \"\"\"Warn when meaning-bearing words present in the original vanish in the rewrite.\"\"\"\n out: list[str] = []\n o, t = original.lower(), transformed.lower()\n\n on, tn = len(_NEGATION_RE.findall(o)), len(_NEGATION_RE.findall(t))\n if on > tn:\n out.append(\n f\"Negation count dropped {on}->{tn}. Verify no claim was inverted or \"\n \"weakened (e.g. 'does not support' -> 'supports').\")\n\n missing_scope = sorted({w for w in _SCOPE_RE.findall(o)} - {w for w in _SCOPE_RE.findall(t)})\n for w in missing_scope:\n out.append(f\"Scope/precision word '{w}' not in output. Verify the claim's scope is unchanged.\")\n\n oc, tc = len(_CONDITIONAL_RE.findall(o)), len(_CONDITIONAL_RE.findall(t))\n if oc > tc:\n out.append(\n f\"Conditional count dropped {oc}->{tc}. Verify a conditional claim \"\n \"wasn't turned into an unconditional one.\")\n\n ow, tw = len(_WEAK_MODAL_RE.findall(o)), len(_WEAK_MODAL_RE.findall(t))\n os, ts = len(_STRONG_MODAL_RE.findall(o)), len(_STRONG_MODAL_RE.findall(t))\n if ow > tw and ts > os:\n out.append(\"Hedged claim may have been strengthened. Verify uncertainty was not turned into certainty.\")\n\n return out\n\n\ndef validate_preservation(\n original_text: str,\n transformed_text: str,\n constraints: list[Constraint] | None = None\n) -> ValidationResult:\n \"\"\"Validate that all constraints are preserved in transformed text.\"\"\"\n\n if constraints is None:\n constraints = extract_constraints(original_text)\n\n missing: list[Constraint] = []\n warnings: list[str] = []\n\n for constraint in constraints:\n if not find_constraint_in_text(constraint, transformed_text):\n missing.append(constraint)\n\n # Generate warnings for near-misses\n for m in missing:\n if m[\"type\"] == \"percentage\":\n # Check if number exists without %\n num = re.search(r'[\\d.]+', m[\"value\"])\n if num and num.group() in transformed_text:\n warnings.append(f\"Number {num.group()} found but missing '%' symbol\")\n\n # Semantic drift warnings. These are NOT numbers/names — they are the\n # meaning-bearing words (negations, scope, conditionals) that the skill's\n # de-hedging rules are most likely to delete and that pure fact-matching\n # cannot see. Surfaced as warnings so the rewrite gets a human re-check;\n # de-slopping legitimately removes some, so they don't hard-fail on their own.\n warnings.extend(semantic_drift_warnings(original_text, transformed_text))\n\n preserved = len(constraints) - len(missing)\n\n return {\n \"passed\": len(missing) == 0,\n \"total_constraints\": len(constraints),\n \"preserved\": preserved,\n \"missing\": missing,\n \"warnings\": warnings\n }\n\n\ndef main() -> None:\n args = sys.argv[1:]\n strict = False\n if args and args[0] == \"--strict\":\n strict = True\n args = args[1:]\n\n if len(args) \u003c 2:\n print(\"Usage: validate_preservation.py \u003coriginal.txt> \u003ctransformed.txt> [constraints.json]\")\n sys.exit(1)\n\n # Read inputs\n try:\n with open(args[0], 'r') as f:\n original_text = f.read()\n with open(args[1], 'r') as f:\n transformed_text = f.read()\n except OSError as e:\n print(json.dumps({\"error\": f\"Could not read input: {e}\"}))\n sys.exit(2)\n\n # Optionally read pre-computed constraints\n constraints = None\n if len(args) > 2:\n with open(args[2], 'r') as f:\n data = json.load(f)\n constraints = data.get(\"constraints\", [])\n\n result = validate_preservation(original_text, transformed_text, constraints)\n\n # Output result\n output = {\n \"passed\": result[\"passed\"],\n \"total_constraints\": result[\"total_constraints\"],\n \"preserved\": result[\"preserved\"],\n \"missing_count\": len(result[\"missing\"]),\n \"missing\": result[\"missing\"],\n \"warnings\": result[\"warnings\"]\n }\n\n print(json.dumps(output, indent=2))\n\n # Exit with appropriate code\n sys.exit(0 if result[\"passed\"] and not (strict and result[\"warnings\"]) else 1)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":14473,"content_sha256":"6fe1c118944a832a99e4dad20ff563c04ff3b12bf2edf617578e31277bfe6ce7"},{"filename":"scripts/voice_card.py","content":"#!/usr/bin/env python3\n\"\"\"Distill a voice profile + samples into a LAYERED, pack-sized style card.\n\nOutput layout (deterministic — same inputs give byte-identical files):\n\n \u003cout>/card.md core sheet, always loaded, \u003c= 300 words, with an\n index table \"when writing X, read card/X.md\"\n \u003cout>/card/\u003csituation>.md one sheet per COVERED situation from the taxonomy\n\nOnly situations with real sample evidence get a sheet. Uncovered situations are\nNAMED in card.md under \"Uncovered\" and never given a fabricated sheet — every\nclaim on every sheet is derived from a measurable profile/sample fact or a\nverbatim sample snippet.\n\n``--coverage`` emits the deterministic lexical coverage matrix (which taxonomy\ndimensions the samples exercise) as JSON and writes nothing. The classifier is\nintentionally coarse: it only DRIVES interactive teach prompts and the sheet\nset. A misclassified sentence can add or drop a sheet, never a card claim, so\nmisclassification is low-stakes by construction.\n\n``--provenance`` also writes \u003cout>/provenance.json (per-sample sha256 + word\ncounts, doc count, genre note, low-confidence flag) so a teach run is auditable.\n\"\"\"\n\nimport argparse\nimport hashlib\nimport json\nimport re\nimport statistics\nimport sys\nfrom pathlib import Path\n\nimport voice_profile\n\n# The situation taxonomy. openings/closings are STRUCTURAL (first/last sentence\n# of every document); the rest are lexical. Order is fixed for determinism.\nTAXONOMY = [\n \"explaining-technical\",\n \"anecdote\",\n \"argument\",\n \"disagreement\",\n \"praise\",\n \"hedging-uncertainty\",\n \"numbers-data\",\n \"addressing-reader\",\n \"openings\",\n \"closings\",\n]\n\nSTRUCTURAL = {\"openings\", \"closings\"}\n\n# Minimum classified sentences before a lexical dimension counts as covered.\nCOVER_THRESHOLD = 2\n\nNUMBER_WORDS = {\n \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\",\n \"ten\", \"eleven\", \"twelve\", \"dozen\", \"hundred\", \"thousand\", \"million\",\n \"twenty\", \"thirty\", \"forty\", \"fifty\", \"sixty\", \"seventy\", \"eighty\",\n \"ninety\", \"percent\", \"half\", \"quarter\", \"double\", \"triple\", \"nine\",\n}\n\n# Lexical signals per dimension. Each entry is a set of lowercase substrings;\n# a sentence matches the dimension if any signal is present (word-boundary\n# aware for single tokens, plain substring for multiword phrases).\nSIGNALS = {\n \"explaining-technical\": {\n \"because\", \"so that\", \"which means\", \"the reason\", \"works by\",\n \"depends on\", \"the way it\", \"in order to\", \"that's how\", \"this is how\",\n \"the trick is\", \"you have to\", \"the point of\",\n },\n \"argument\": {\n \"therefore\", \"thus\", \"consequently\", \"the point is\", \"i distrust\",\n \"clearly\", \"obviously\", \"in fact\", \"the truth is\", \"matters because\",\n \"that's the point\", \"either way\", \"that's rare\", \"nobody\", \"no one\",\n },\n \"disagreement\": {\n \"but i\", \"i don't\", \"no one\", \"nobody\", \"wrong\", \"i distrust\",\n \"rather than\", \"not because\", \"i hate\", \"don't trust\", \"disagree\",\n \"however\", \"i wasn't\", \"makes sense\", \"i can't\",\n },\n \"praise\": {\n \"extraordinary\", \"wonderful\", \"dependable\", \"lovely\", \"beautiful\",\n \"kindly\", \"generous\", \"grateful\", \"delightful\", \"plenty\", \"good choice\",\n \"with care\", \"extraordinary care\", \"almost pleasant\", \"i like\",\n },\n \"hedging-uncertainty\": {\n \"maybe\", \"perhaps\", \"probably\", \"might\", \"i guess\", \"i think\",\n \"seems\", \"sort of\", \"kind of\", \"possibly\", \"i suppose\", \"not sure\",\n \"or won't\", \"or it won't\", \"i believed\", \"or maybe\",\n },\n \"addressing-reader\": set(), # handled specially (2nd person / question)\n \"numbers-data\": set(), # handled specially (digits / number words)\n \"anecdote\": set(), # handled specially (1st person + past/time cue)\n}\n\nFIRST_PERSON = {\"i\", \"we\", \"my\", \"me\", \"we've\", \"i've\", \"i'll\", \"i'd\", \"our\"}\nTIME_CUES = {\n \"today\", \"yesterday\", \"tonight\", \"morning\", \"evening\", \"night\", \"ago\",\n \"monday\", \"tuesday\", \"wednesday\", \"thursday\", \"friday\", \"saturday\", \"sunday\",\n \"last\", \"once\", \"then\", \"later\", \"week\", \"year\",\n}\n\n\ndef sha256_file(path):\n return hashlib.sha256(path.read_bytes()).hexdigest()\n\n\ndef split_sentences_text(text):\n \"\"\"Split into trimmed sentence strings, preserving original wording.\"\"\"\n out = []\n for chunk in re.split(r\"(?\u003c=[.!?])\\s+\", text.strip()):\n chunk = re.sub(r\"\\s+\", \" \", chunk).strip()\n if chunk:\n out.append(chunk)\n return out\n\n\ndef _has_word(sentence_low, token):\n return re.search(r\"\\b\" + re.escape(token) + r\"\\b\", sentence_low) is not None\n\n\ndef _matches(sentence_low, signals):\n for sig in signals:\n if \" \" in sig:\n if sig in sentence_low:\n return True\n elif _has_word(sentence_low, sig):\n return True\n return False\n\n\ndef _is_numbers(sentence_low):\n if re.search(r\"\\d\", sentence_low):\n return True\n return any(_has_word(sentence_low, w) for w in NUMBER_WORDS)\n\n\ndef _is_anecdote(sentence_low):\n toks = set(voice_profile.words(sentence_low))\n if not (toks & FIRST_PERSON):\n return False\n if toks & TIME_CUES:\n return True\n return any(t.endswith(\"ed\") and len(t) > 3 for t in toks)\n\n\ndef _is_addressing(sentence):\n low = sentence.lower()\n if sentence.rstrip().endswith(\"?\"):\n return True\n return _has_word(low, \"you\") or _has_word(low, \"your\") or _has_word(low, \"you're\")\n\n\ndef classify_dimension(sentence):\n \"\"\"Return the set of lexical dimensions a single sentence exercises.\"\"\"\n low = sentence.lower()\n hit = set()\n for dim, signals in SIGNALS.items():\n if dim in (\"addressing-reader\", \"numbers-data\", \"anecdote\"):\n continue\n if _matches(low, signals):\n hit.add(dim)\n if _is_numbers(low):\n hit.add(\"numbers-data\")\n if _is_addressing(sentence):\n hit.add(\"addressing-reader\")\n if _is_anecdote(low):\n hit.add(\"anecdote\")\n return hit\n\n\ndef collect(samples_dir):\n \"\"\"Return (docs, sentences) where each doc is a list of sentence strings.\"\"\"\n docs = []\n for path in voice_profile.iter_docs(samples_dir):\n sents = split_sentences_text(path.read_text(errors=\"replace\"))\n if sents:\n docs.append(sents)\n return docs\n\n\ndef coverage_matrix(docs):\n \"\"\"Deterministic coverage of the taxonomy over the sample sentences.\"\"\"\n buckets = {dim: [] for dim in TAXONOMY}\n for doc in docs:\n if doc:\n buckets[\"openings\"].append(doc[0])\n buckets[\"closings\"].append(doc[-1])\n for sent in doc:\n for dim in classify_dimension(sent):\n buckets[dim].append(sent)\n matrix = {}\n for dim in TAXONOMY:\n sents = buckets[dim]\n if dim in STRUCTURAL:\n covered = len(docs) >= 1\n else:\n covered = len(sents) >= COVER_THRESHOLD\n matrix[dim] = {\n \"count\": len(sents),\n \"covered\": covered,\n \"structural\": dim in STRUCTURAL,\n }\n return matrix, buckets\n\n\ndef _shortest(sentences, k=3):\n ordered = sorted(set(sentences), key=lambda s: (len(s), s))\n return ordered[:k]\n\n\ndef _contraction_examples(docs, k=3):\n seen = {}\n for doc in docs:\n for sent in doc:\n for tok in voice_profile.words(sent):\n if \"'\" in tok:\n seen[tok] = seen.get(tok, 0) + 1\n ranked = sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))\n return [w for w, _ in ranked[:k]]\n\n\ndef _opener_words(docs, k=5):\n counts = {}\n for doc in docs:\n for sent in doc:\n toks = voice_profile.words(sent)\n if toks:\n counts[toks[0]] = counts.get(toks[0], 0) + 1\n ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))\n return [w for w, _ in ranked[:k]]\n\n\ndef _burstiness(profile):\n med = profile[\"sentence_lengths\"][\"median\"]\n iqr = profile[\"sentence_lengths\"][\"iqr\"]\n if med \u003c= 0:\n return \"uneven\"\n ratio = iqr / med\n if ratio \u003c 0.5:\n return \"steady\"\n if ratio \u003c 1.1:\n return \"moderately bursty\"\n return \"very bursty\"\n\n\ndef _numbers_tokens(sentences):\n toks = set()\n for sent in sentences:\n for m in re.findall(r\"\\d[\\d,:.]*\", sent):\n toks.add(m)\n for w in voice_profile.words(sent.lower()):\n if w in NUMBER_WORDS:\n toks.add(w)\n return sorted(toks)\n\n\ndef _sheet_markers(dim, sentences, profile):\n lengths = [len(voice_profile.words(s)) for s in sentences]\n med = statistics.median(lengths) if lengths else 0\n lines = [f\"- Sentences in samples exercising this: {len(sentences)}.\",\n f\"- Median length of those sentences: {int(med)} words.\"]\n if dim == \"numbers-data\":\n toks = _numbers_tokens(sentences)\n lines.append(\"- Numeric tokens actually used: \" + \", \".join(toks[:12]) + \".\")\n elif dim == \"addressing-reader\":\n q = sum(1 for s in sentences if s.rstrip().endswith(\"?\"))\n lines.append(f\"- Direct questions to the reader: {q}.\")\n lines.append(\"- Second person appears; keep it plain, no salesy 'you'.\")\n elif dim == \"openings\":\n openers = sorted({voice_profile.words(s)[0] for s in sentences if voice_profile.words(s)})\n lines.append(\"- Documents open on: \" + \", \".join(openers[:8]) + \".\")\n elif dim == \"closings\":\n lines.append(\"- Endings land on a concrete image, not a moral recap.\")\n else:\n cr = profile[\"contraction_rate\"]\n lines.append(f\"- Overall contraction rate: {cr:.3f} (keep it consistent here).\")\n return lines\n\n\nSHEET_HOWTO = {\n \"explaining-technical\": \"Explain by naming the concrete mechanism, not the abstraction. Short causal sentences; 'because' does the work.\",\n \"anecdote\": \"Tell it first person, past tense, one scene at a time. Concrete nouns, sensory detail, no summarizing moral.\",\n \"argument\": \"State the claim flat, then the reason. No hedging scaffold; the point lands in one line.\",\n \"disagreement\": \"Disagree by contrast, not confrontation. 'I don't', 'rather than', a plain preference rather than a takedown.\",\n \"praise\": \"Praise through specific, restrained detail. Understated approval, never gushing.\",\n \"hedging-uncertainty\": \"Hold uncertainty lightly with 'maybe' / 'probably' / 'or it won't', not corporate qualifiers.\",\n \"numbers-data\": \"Numbers stay small, concrete, woven into the scene rather than tabulated.\",\n \"addressing-reader\": \"Address the reader sparingly and plainly; a direct question or a flat 'you can'.\",\n \"openings\": \"Open cold on a concrete fact or action. No throat-clearing, no thesis statement.\",\n \"closings\": \"Close on a small, specific image. No wrap-up, no 'ultimately'.\",\n}\n\n\ndef build_sheet(dim, sentences, profile):\n title = dim.replace(\"-\", \" \")\n lines = [f\"# Voice sheet: {title}\", \"\"]\n lines.append(SHEET_HOWTO[dim])\n lines.append(\"\")\n lines.append(\"## Sample snippets\")\n for snip in _shortest(sentences):\n lines.append(f\"> {snip}\")\n lines.append(\"\")\n lines.append(\"## Measured markers\")\n lines.extend(_sheet_markers(dim, sentences, profile))\n lines.append(\"\")\n return \"\\n\".join(lines) + \"\\n\"\n\n\ndef _never_does(profile):\n never = []\n p = profile[\"punctuation\"]\n label = {\";\": \"semicolons\", \"!\": \"exclamation points\", \":\": \"colons\",\n \"-\": \"hyphenated dashes\", \"(\": \"parentheticals\"}\n for mark, name in label.items():\n if p.get(mark, 0.0) == 0.0:\n never.append(name)\n return never\n\n\ndef build_card(profile, docs, matrix, name):\n med = int(profile[\"sentence_lengths\"][\"median\"])\n iqr = int(profile[\"sentence_lengths\"][\"iqr\"])\n contr = profile[\"contraction_rate\"]\n examples = _contraction_examples(docs)\n openers = _opener_words(docs)\n never = _never_does(profile)\n covered = [d for d in TAXONOMY if matrix[d][\"covered\"]]\n uncovered = [d for d in TAXONOMY if not matrix[d][\"covered\"]]\n\n lines = [f\"# Voice card: {name}\", \"\"]\n lines.append(\n f\"Rhythm: median sentence {med} words, IQR {iqr}, {_burstiness(profile)}. \"\n f\"Paragraphs average {int(profile['paragraph_stats']['mean_words'])} words.\"\n )\n if examples:\n lines.append(\n f\"Contractions: rate {contr:.3f}; e.g. \" + \", \".join(examples) + \".\"\n )\n else:\n lines.append(f\"Contractions: rate {contr:.3f}; rarely contracts.\")\n if never:\n lines.append(\"Never: \" + \"; \".join(never) + \".\")\n lines.append(\"Openers: \" + \", \".join(openers) + \".\")\n lines.append(\"\")\n lines.append(\"Match rhythm and habits first; keep facts and meaning intact.\")\n lines.append(\"\")\n lines.append(\"## When writing, read the matching sheet\")\n lines.append(\"\")\n lines.append(\"| Situation | Sheet |\")\n lines.append(\"|-----------|-------|\")\n for dim in covered:\n lines.append(f\"| {dim.replace('-', ' ')} | card/{dim}.md |\")\n lines.append(\"\")\n if uncovered:\n lines.append(\n \"Uncovered (no sample evidence — do not fabricate a voice for these): \"\n + \", \".join(uncovered) + \".\"\n )\n return \"\\n\".join(lines) + \"\\n\"\n\n\ndef card_word_count(card_text):\n return len(re.findall(r\"[A-Za-z0-9']+\", card_text))\n\n\ndef write_card(profile, samples_dir, out_dir, name):\n docs = collect(samples_dir)\n matrix, buckets = coverage_matrix(docs)\n out = Path(out_dir)\n (out / \"card\").mkdir(parents=True, exist_ok=True)\n # Remove any stale sheets so uncovered dims never keep an old file.\n for stale in (out / \"card\").glob(\"*.md\"):\n stale.unlink()\n card = build_card(profile, docs, matrix, name)\n (out / \"card.md\").write_text(card)\n for dim in TAXONOMY:\n if matrix[dim][\"covered\"]:\n (out / \"card\" / f\"{dim}.md\").write_text(build_sheet(dim, buckets[dim], profile))\n return matrix\n\n\ndef write_provenance(profile, samples_dir, out_dir):\n samples = []\n total = 0\n for path in voice_profile.iter_docs(samples_dir):\n text = path.read_text(errors=\"replace\")\n wc = len(voice_profile.words(text))\n total += wc\n samples.append({\n \"file\": path.name,\n \"sha256\": sha256_file(path),\n \"words\": wc,\n })\n meta = profile.get(\"metadata\", {})\n prov = {\n \"doc_count\": len(samples),\n \"total_words\": total,\n \"samples\": samples,\n \"genre_note\": meta.get(\"genre_warning\", \"\") or \"same-genre samples assumed\",\n \"low_confidence\": bool(meta.get(\"low_confidence\", total \u003c 2000)),\n }\n Path(out_dir, \"provenance.json\").write_text(\n json.dumps(prov, indent=2, sort_keys=True) + \"\\n\"\n )\n return prov\n\n\ndef profile_mismatch(supplied, recomputed, path=\"\"):\n \"\"\"First named field where the supplied profile disagrees with one recomputed\n from --samples, or None. Counts (ints) compare exactly; floats within 1e-6.\n The function-word background is a normalizer (from --background, not the\n sample content), so it is not part of the equality check.\"\"\"\n here = path or \"\u003croot>\"\n if isinstance(supplied, bool) or isinstance(recomputed, bool):\n return None if supplied == recomputed else here\n if isinstance(supplied, dict):\n if not isinstance(recomputed, dict):\n return here\n for k in sorted(set(supplied) | set(recomputed)):\n if k == \"function_word_background\":\n continue\n if k not in supplied or k not in recomputed:\n return f\"{path}.{k}\".lstrip(\".\")\n m = profile_mismatch(supplied[k], recomputed[k], f\"{path}.{k}\".lstrip(\".\"))\n if m:\n return m\n return None\n if isinstance(supplied, list):\n if not isinstance(recomputed, list) or len(supplied) != len(recomputed):\n return here\n for i, (x, y) in enumerate(zip(supplied, recomputed)):\n m = profile_mismatch(x, y, f\"{path}[{i}]\")\n if m:\n return m\n return None\n if isinstance(supplied, int) and isinstance(recomputed, int):\n return None if supplied == recomputed else here\n if isinstance(supplied, (int, float)) and isinstance(recomputed, (int, float)):\n return None if abs(supplied - recomputed) \u003c= 1e-6 else here\n return None if supplied == recomputed else here\n\n\ndef parse_args(argv):\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\"--profile\", required=True)\n parser.add_argument(\"--samples\", required=True)\n parser.add_argument(\"--out\")\n parser.add_argument(\"--name\", default=\"voice\")\n parser.add_argument(\"--coverage\", action=\"store_true\",\n help=\"print the coverage matrix as JSON and write nothing\")\n parser.add_argument(\"--provenance\", action=\"store_true\",\n help=\"also write \u003cout>/provenance.json\")\n return parser.parse_args(argv)\n\n\ndef main(argv):\n args = parse_args(argv)\n profile_path = Path(args.profile)\n samples_dir = Path(args.samples)\n if not profile_path.exists() or not samples_dir.is_dir():\n print(\"missing profile or samples dir\", file=sys.stderr)\n return 2\n if not list(voice_profile.iter_docs(samples_dir)):\n print(f\"no sample documents in {samples_dir}: teach reads only .txt and .md \"\n f\"files (recursively). Rename samples to .txt/.md or point --samples at \"\n f\"the right directory.\", file=sys.stderr)\n return 2\n profile = json.loads(profile_path.read_text())\n\n # Consistency gate: the supplied profile must describe these very samples.\n recomputed = voice_profile.build_profile(samples_dir)\n mismatch = profile_mismatch(profile, recomputed)\n if mismatch is not None:\n print(f\"profile does not match --samples (recompute differs at '{mismatch}'); \"\n f\"rebuild the profile from these samples with voice_profile.py\",\n file=sys.stderr)\n return 2\n\n if args.coverage:\n docs = collect(samples_dir)\n matrix, _ = coverage_matrix(docs)\n print(json.dumps(matrix, indent=2, sort_keys=True))\n return 0\n\n if not args.out:\n print(\"--out is required unless --coverage\", file=sys.stderr)\n return 2\n write_card(profile, samples_dir, args.out, args.name)\n if args.provenance:\n write_provenance(profile, samples_dir, args.out)\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main(sys.argv[1:]))\n","content_type":"text/x-python; charset=utf-8","language":"python","size":18548,"content_sha256":"1a444356aa74d5b5a863be638926490da56eeeaede5d73b973d20fcc8aa62a85"},{"filename":"scripts/voice_profile.py","content":"#!/usr/bin/env python3\n\"\"\"Build deterministic stylometric voice profiles.\n\nThe built-in background table is a compact Writeprints-style English baseline:\ncommon function words seeded from public-domain frequency lists used by classic\nauthorship-attribution examples, with broad fallback means/stddevs for words not\nobserved in a caller-supplied background corpus. It is meant only as a stable\nnormalizer; pass --background with same-genre documents for calibrated work.\n\"\"\"\n\nimport argparse\nimport collections\nimport json\nimport math\nimport re\nimport statistics\nimport sys\nfrom pathlib import Path\n\nFUNCTION_WORDS = \"\"\"\nthe of and to in a is that it for as with was on be by he i this are or his from at\nwhich but have an had they you were their one all we can her has there been if more\nwhen will would who so no she about out up into do any your what than them some could\nthese other then its our two may first my now such like over only also after most did\nmany before must through back where much should well people down own just because good\neach those how under see made very being make between both even another while last\nmight still same never every against since off though yet without within upon among\nuntil during per either neither nor whether whose whom why again once here there\ntherefore however although nevertheless moreover instead indeed perhaps rather thus\nelse already almost around across behind beyond near toward towards above below beside\ninside outside along plus minus except despite via versus including regarding concerning\nam were does done doing having let lets cannot dont didn't doesn't isn't aren't wasn't\nweren't haven't hasn't hadn't won't wouldn't shouldn't couldn't mightn't mustn't i'm\nyou're he's she's it's we're they're i've you've we've they've i'd you'd he'd she'd we'd\nthey'd i'll you'll he'll she'll we'll they'll me him us mine yours ours theirs myself\nyourself himself herself itself ourselves yourselves themselves\n\"\"\".split()\n\nPUNCT = [\",\", \".\", \";\", \":\", \"?\", \"!\", \"-\", \"(\", \")\", '\"', \"'\"]\nWORD_RE = re.compile(r\"[A-Za-z]+(?:'[A-Za-z]+)?|\\d+\")\nSENT_RE = re.compile(r\"[^.!?]+[.!?]?\")\n\n\ndef iter_docs(root):\n for path in sorted(Path(root).rglob(\"*\")):\n if path.suffix.lower() in {\".txt\", \".md\"} and path.is_file():\n yield path\n\n\ndef normalize(text):\n return re.sub(r\"\\s+\", \" \", text.lower()).strip()\n\n\ndef words(text):\n return WORD_RE.findall(text.lower())\n\n\ndef sentences(text):\n out = []\n for part in SENT_RE.findall(text):\n toks = words(part)\n if toks:\n out.append(toks)\n return out\n\n\ndef char3_counts(text, limit=None):\n norm = normalize(text)\n grams = collections.Counter(norm[i:i + 3] for i in range(max(0, len(norm) - 2)))\n items = sorted(grams.items(), key=lambda kv: (-kv[1], kv[0]))\n if limit:\n items = items[:limit]\n return dict(items)\n\n\ndef function_freq(tokens):\n total = max(1, len(tokens))\n counts = collections.Counter(tokens)\n return {w: counts[w] / total for w in FUNCTION_WORDS}\n\n\ndef sentence_stats(text):\n lengths = [len(s) for s in sentences(text)]\n if not lengths:\n return {\"lengths\": [], \"median\": 0.0, \"iqr\": 0.0}\n ordered = sorted(lengths)\n mid = statistics.median(ordered)\n q1 = statistics.median(ordered[:len(ordered) // 2] or ordered)\n q3 = statistics.median(ordered[(len(ordered) + 1) // 2:] or ordered)\n return {\"lengths\": lengths, \"median\": mid, \"iqr\": q3 - q1}\n\n\ndef mtld(tokens, threshold=0.72):\n if len(tokens) \u003c 20:\n return 0.0\n factors = 0.0\n types = set()\n count = 0\n for tok in tokens:\n count += 1\n types.add(tok)\n if len(types) / count \u003c= threshold:\n factors += 1\n types.clear()\n count = 0\n if count:\n ttr = len(types) / count\n factors += (1 - ttr) / (1 - threshold) if threshold \u003c 1 else 0\n return len(tokens) / factors if factors else float(len(tokens))\n\n\ndef feature_bundle(text):\n toks = words(text)\n total = max(1, len(toks))\n punct_counts = collections.Counter(ch for ch in text if ch in PUNCT)\n contractions = sum(1 for t in toks if \"'\" in t)\n hist = collections.Counter(min(len(t), 15) for t in toks)\n paragraphs = [p for p in re.split(r\"\\n\\s*\\n\", text.strip()) if p.strip()]\n return {\n \"char3\": char3_counts(text, 2000),\n \"function_words\": function_freq(toks),\n \"sentence_lengths\": sentence_stats(text),\n \"punctuation\": {p: punct_counts[p] / total for p in PUNCT},\n \"contraction_rate\": contractions / total,\n \"mtld\": mtld(toks),\n \"word_length_histogram\": {str(i): hist[i] / total for i in range(1, 16)},\n \"paragraph_stats\": {\n \"count\": len(paragraphs),\n \"mean_words\": (sum(len(words(p)) for p in paragraphs) / len(paragraphs)) if paragraphs else 0.0,\n },\n \"total_words\": len(toks),\n }\n\n\ndef background_stats(root=None):\n docs = []\n if root:\n docs = [p.read_text(errors=\"replace\") for p in iter_docs(root)]\n if not docs:\n return {w: {\"mean\": 0.0025 if w not in {\"the\", \"of\", \"and\", \"to\", \"in\", \"a\"} else 0.025,\n \"std\": 0.006} for w in FUNCTION_WORDS}\n rows = [function_freq(words(text)) for text in docs]\n stats = {}\n for w in FUNCTION_WORDS:\n vals = [r[w] for r in rows]\n stats[w] = {\n \"mean\": statistics.mean(vals),\n \"std\": statistics.pstdev(vals) or 0.0001,\n }\n return stats\n\n\ndef build_profile(samples_dir, background=None):\n paths = list(iter_docs(samples_dir))\n text = \"\\n\\n\".join(p.read_text(errors=\"replace\") for p in paths)\n profile = feature_bundle(text)\n profile[\"function_word_background\"] = background_stats(background)\n profile[\"metadata\"] = {\n \"doc_count\": len(paths),\n \"total_words\": profile[\"total_words\"],\n \"low_confidence\": profile[\"total_words\"] \u003c 2000,\n \"genre_warning\": \"profile has fewer than 2000 words\" if profile[\"total_words\"] \u003c 2000 else \"\",\n }\n return profile\n\n\ndef parse_args(argv):\n parser = argparse.ArgumentParser()\n parser.add_argument(\"samples_dir\")\n parser.add_argument(\"-o\", \"--output\", required=True)\n parser.add_argument(\"--background\")\n return parser.parse_args(argv)\n\n\ndef main(argv):\n args = parse_args(argv)\n root = Path(args.samples_dir)\n if not root.is_dir():\n print(f\"missing samples dir: {root}\", file=sys.stderr)\n return 2\n if not list(iter_docs(root)):\n print(f\"no sample documents in {root}: only .txt and .md files are read \"\n f\"(recursively). Rename samples to .txt/.md or point at the right \"\n f\"directory.\", file=sys.stderr)\n return 2\n profile = build_profile(root, args.background)\n if profile[\"metadata\"][\"low_confidence\"]:\n print(profile[\"metadata\"][\"genre_warning\"], file=sys.stderr)\n Path(args.output).write_text(json.dumps(profile, indent=2, sort_keys=True) + \"\\n\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main(sys.argv[1:]))\n","content_type":"text/x-python; charset=utf-8","language":"python","size":7083,"content_sha256":"8ba1af055cc8e7c7e34ae4802939a20192746fde0b7a86c3a644a738b3764bf0"},{"filename":"scripts/voice_score.py","content":"#!/usr/bin/env python3\n\"\"\"Score a candidate against a voice profile.\n\nLower composite means more user-like. The composite is half GI-rank penalty and\nhalf clipped, weighted impostor z-distance using the WP10a research weights.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport json\nimport math\nimport random\nimport statistics\nimport sys\nfrom pathlib import Path\n\nimport voice_profile\n\nWEIGHTS = {\n \"char3\": 0.30,\n \"delta\": 0.25,\n \"sentence_emd\": 0.10,\n \"punctuation\": 0.08,\n \"contraction\": 0.07,\n \"mtld\": 0.10,\n \"word_length\": 0.10,\n}\n\n\ndef cosine_distance(a, b, keys=None):\n keys = list(keys) if keys is not None else sorted(set(a) | set(b))\n dot = sum(a.get(k, 0.0) * b.get(k, 0.0) for k in keys)\n na = math.sqrt(sum(a.get(k, 0.0) ** 2 for k in keys))\n nb = math.sqrt(sum(b.get(k, 0.0) ** 2 for k in keys))\n if not na or not nb:\n return 1.0\n return 1 - (dot / (na * nb))\n\n\ndef z_function_vector(freqs, bg, top):\n keys = sorted(bg, key=lambda k: bg[k].get(\"mean\", 0), reverse=True)[:top]\n return {k: (freqs.get(k, 0.0) - bg[k][\"mean\"]) / (bg[k][\"std\"] or 0.0001) for k in keys}\n\n\ndef emd(a, b):\n if not a or not b:\n return 0.0\n max_len = max(max(a), max(b))\n ca = cb = dist = 0.0\n for i in range(1, max_len + 1):\n ca += sum(1 for x in a if x == i) / len(a)\n cb += sum(1 for x in b if x == i) / len(b)\n dist += abs(ca - cb)\n return dist\n\n\ndef l1(a, b, keys):\n return sum(abs(a.get(k, 0.0) - b.get(k, 0.0)) for k in keys)\n\n\ndef distances(profile, feats, subset=None):\n subset = set(subset or WEIGHTS)\n out = {}\n if \"char3\" in subset:\n keys = set(profile[\"char3\"]) | set(feats[\"char3\"])\n out[\"char3\"] = cosine_distance(profile[\"char3\"], feats[\"char3\"], keys)\n if \"delta\" in subset:\n top = 50 if feats[\"total_words\"] \u003c 300 else 200\n bg = profile[\"function_word_background\"]\n pv = z_function_vector(profile[\"function_words\"], bg, top)\n cv = z_function_vector(feats[\"function_words\"], bg, top)\n out[\"delta\"] = cosine_distance(pv, cv, pv.keys())\n if \"sentence_emd\" in subset:\n med = profile[\"sentence_lengths\"].get(\"median\") or 1.0\n out[\"sentence_emd\"] = emd(profile[\"sentence_lengths\"][\"lengths\"], feats[\"sentence_lengths\"][\"lengths\"]) / med\n if \"punctuation\" in subset:\n out[\"punctuation\"] = l1(profile[\"punctuation\"], feats[\"punctuation\"], voice_profile.PUNCT)\n if \"contraction\" in subset:\n out[\"contraction\"] = abs(profile[\"contraction_rate\"] - feats[\"contraction_rate\"])\n if \"mtld\" in subset:\n out[\"mtld\"] = abs(profile[\"mtld\"] - feats[\"mtld\"]) / (profile[\"mtld\"] or 1.0)\n if \"word_length\" in subset:\n keys = [str(i) for i in range(1, 16)]\n out[\"word_length\"] = l1(profile[\"word_length_histogram\"], feats[\"word_length_histogram\"], keys)\n return out\n\n\ndef weighted_sum(dists):\n return sum(WEIGHTS[k] * dists.get(k, 0.0) for k in WEIGHTS)\n\n\ndef impostor_features(root):\n feats = []\n for path in voice_profile.iter_docs(root):\n feats.append((str(path), voice_profile.feature_bundle(path.read_text(errors=\"replace\"))))\n return feats\n\n\ndef zscores(candidate, impostor_rows):\n out = {}\n for key in WEIGHTS:\n vals = [row[key] for row in impostor_rows if key in row]\n if key not in candidate or not vals:\n out[key] = None\n continue\n mean = statistics.mean(vals)\n std = statistics.pstdev(vals) or 0.0001\n out[key] = (candidate[key] - mean) / std\n return out\n\n\ndef gi_score(profile, cand_feats, impostors, seed):\n rng = random.Random(seed)\n keys = list(WEIGHTS)\n wins = 0\n trials = 64\n # Per-key distances depend only on (profile, feats) -- not on which subset\n # of keys a given trial happens to draw -- so compute every key's distance\n # ONCE per candidate/impostor here, then have each trial do a subset-\n # weighted sum over the precomputed values instead of recomputing\n # distances() from scratch on every trial. Arithmetically exact: the\n # trial loop still consumes rng.random()/rng.sample() in the same order,\n # and summing WEIGHTS[k]*value only over the drawn subset is the same\n # float sequence weighted_sum(distances(..., subset)) produced (excluded\n # keys contributed an exact 0.0 term either way).\n cand_dists = distances(profile, cand_feats)\n impostor_dists = [(name, distances(profile, imp)) for name, imp in impostors]\n for _ in range(trials):\n subset = [k for k in keys if rng.random() \u003c 0.5] or [rng.choice(keys)]\n cand = sum(WEIGHTS[k] * cand_dists.get(k, 0.0) for k in subset)\n sampled = rng.sample(impostor_dists, k=min(len(impostor_dists), max(1, len(impostor_dists) // 2)))\n if all(cand \u003c sum(WEIGHTS[k] * imp_dists.get(k, 0.0) for k in subset)\n for _, imp_dists in sampled):\n wins += 1\n return wins / trials\n\n\ndef ngrams(tokens, n=4):\n return set(tuple(tokens[i:i + n]) for i in range(max(0, len(tokens) - n + 1)))\n\n\nLCS_THRESHOLD = 120\n\n\ndef has_common_substring_over(a: str, b: str, min_length: int) -> bool:\n \"\"\"Rolling-hash check: do ``a`` and ``b`` share a contiguous substring of at\n least ``min_length`` characters?\n\n O(len(a) + len(b)) expected time, vs. the O(len(a) * len(b)) classic DP a\n true longest-common-substring computation needs. This only answers the\n threshold question the copy gate actually asks (\"is there a shared run\n longer than N chars\"); it does not recover the true longest common\n substring length. Every hash match is verified against the source\n characters before being trusted, so a hash collision never produces a\n false positive.\n \"\"\"\n if min_length \u003c= 0:\n return bool(a) and bool(b)\n if len(a) \u003c min_length or len(b) \u003c min_length:\n return False\n\n base = 257\n mod = (1 \u003c\u003c 61) - 1\n high_power = pow(base, min_length - 1, mod)\n\n def window_hashes(s: str) -> dict[int, list[int]]:\n table: dict[int, list[int]] = {}\n h = 0\n for i in range(min_length):\n h = (h * base + ord(s[i])) % mod\n table.setdefault(h, []).append(0)\n for i in range(min_length, len(s)):\n h = ((h - ord(s[i - min_length]) * high_power) * base + ord(s[i])) % mod\n table.setdefault(h, []).append(i - min_length + 1)\n return table\n\n table_a = window_hashes(a)\n table_b = window_hashes(b)\n for h, starts_b in table_b.items():\n starts_a = table_a.get(h)\n if not starts_a:\n continue\n for sb in starts_b:\n window_b = b[sb:sb + min_length]\n for sa in starts_a:\n if a[sa:sa + min_length] == window_b:\n return True\n return False\n\n\ndef copy_gate(candidate_text, samples_dir):\n cand_grams = ngrams(voice_profile.words(candidate_text))\n max_overlap = 0.0\n lcs_violation = False\n for path in voice_profile.iter_docs(samples_dir):\n text = path.read_text(errors=\"replace\")\n sample_grams = ngrams(voice_profile.words(text))\n if cand_grams:\n max_overlap = max(max_overlap, len(cand_grams & sample_grams) / len(cand_grams))\n if has_common_substring_over(candidate_text, text, LCS_THRESHOLD + 1):\n lcs_violation = True\n # \"longest_common_substring\" is reported for backward-compatible shape only:\n # nothing pins its exact value (checked evals/adversarial-evals.json and\n # evals/check_*.py for max_lcs/longest_common_substring assertions; none\n # exist). Its semantics changed from \"the true LCS length\" to \"the\n # matched threshold window length when a violation-length run was found,\n # else 0\" -- computing the true max cheaply isn't possible without the\n # same O(n*m) cost this function exists to avoid.\n max_lcs = LCS_THRESHOLD + 1 if lcs_violation else 0\n return {\n \"max_overlap\": max_overlap,\n \"longest_common_substring\": max_lcs,\n \"violation\": max_overlap > 0.35 or lcs_violation,\n }\n\n\ndef parse_args(argv):\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--profile\", required=True)\n parser.add_argument(\"--impostors\", required=True)\n parser.add_argument(\"--seed\", required=True, type=int)\n parser.add_argument(\"--samples\")\n parser.add_argument(\"candidate_file\")\n return parser.parse_args(argv)\n\n\ndef read_candidate(path):\n if path == \"-\":\n return sys.stdin.buffer.read().decode(\"utf-8\", errors=\"replace\")\n p = Path(path)\n if not p.exists():\n raise FileNotFoundError(path)\n return p.read_text(errors=\"replace\")\n\n\ndef main(argv):\n args = parse_args(argv)\n if not Path(args.profile).exists() or not Path(args.impostors).is_dir():\n print(\"missing profile or impostors\", file=sys.stderr)\n return 2\n try:\n text = read_candidate(args.candidate_file)\n except FileNotFoundError as e:\n print(f\"missing candidate: {e}\", file=sys.stderr)\n return 2\n profile = json.loads(Path(args.profile).read_text())\n feats = voice_profile.feature_bundle(text)\n low = feats[\"total_words\"] \u003c 150\n impostors = impostor_features(args.impostors)\n dist = distances(profile, feats)\n imp_dist = [distances(profile, f) for _, f in impostors]\n zs = zscores(dist, imp_dist)\n zsum = sum(WEIGHTS[k] * max(-3.0, min(3.0, zs[k] if zs[k] is not None else 0.0)) for k in WEIGHTS)\n gi = gi_score(profile, feats, impostors, args.seed) if impostors else 0.0\n result = {\n \"candidate_words\": feats[\"total_words\"],\n \"low_confidence\": low,\n \"distances\": {k: (None if low and k in {\"sentence_emd\", \"mtld\"} else v) for k, v in dist.items()},\n \"z_scores\": {k: (None if low and k in {\"sentence_emd\", \"mtld\"} else v) for k, v in zs.items()},\n \"gi\": gi,\n \"composite\": 0.5 * (1 - gi) + 0.5 * zsum,\n }\n if args.samples:\n result[\"copy_gate\"] = copy_gate(text, args.samples)\n print(json.dumps(result, indent=2, sort_keys=True))\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main(sys.argv[1:]))\n","content_type":"text/x-python; charset=utf-8","language":"python","size":10135,"content_sha256":"b3281bf6e545c207827b130c9d69d0706258f3c4e15f0abec61124b99a25af57"},{"filename":"scripts/wiki_sync.py","content":"#!/usr/bin/env python3\n\"\"\"\nSync unslop rules with Wikipedia's \"Signs of AI writing\" page.\n\nFetches the latest revision, parses structured data from wikitext,\nand outputs diffs or integration prompts for Claude Code.\n\nSubcommands:\n check - Check for updates (exits 0 if no updates, 1 if updates available)\n diff - Output structured JSON diff of changes since last sync\n prompt - Output a Claude Code integration prompt with mapped changes\n\nUsage:\n python wiki_sync.py check\n python wiki_sync.py diff\n python wiki_sync.py prompt\n python wiki_sync.py prompt | claude -p --allowedTools Edit,Read,Grep,Glob,Bash\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport hashlib\nimport json\nimport re\nimport sys\nimport urllib.request\nimport urllib.parse\nfrom pathlib import Path\nfrom typing import TypedDict\n\nSTATE_FILE = Path(__file__).parent / \".wiki_sync_state.json\"\nWIKI_PAGE = \"Wikipedia:Signs_of_AI_writing\"\nAPI_URL = \"https://en.wikipedia.org/w/api.php\"\n\n\nclass SyncState(TypedDict):\n last_revision_id: int\n last_timestamp: str\n content_hash: str\n last_content_snapshot: str\n\n\nclass ParsedSection(TypedDict):\n title: str\n level: int\n content: str\n watch_words: list[str]\n examples: list[str]\n\n\nclass Change(TypedDict):\n type: str # \"new_section\", \"new_words\", \"modified\", \"removed\"\n section: str\n details: str\n words: list[str]\n\n\ndef fetch_latest_revision() -> tuple[int, str, str]:\n \"\"\"Fetch the latest revision content from Wikipedia.\n\n Returns (revision_id, timestamp, wikitext).\n \"\"\"\n params = {\n \"action\": \"query\",\n \"titles\": WIKI_PAGE,\n \"prop\": \"revisions\",\n \"rvslots\": \"*\",\n \"rvprop\": \"ids|timestamp|content\",\n \"format\": \"json\",\n \"formatversion\": \"2\",\n }\n url = f\"{API_URL}?{urllib.parse.urlencode(params)}\"\n req = urllib.request.Request(url, headers={\"User-Agent\": \"unslop-wiki-sync/1.0\"})\n\n with urllib.request.urlopen(req, timeout=30) as resp:\n data = json.loads(resp.read().decode(\"utf-8\"))\n\n page = data[\"query\"][\"pages\"][0]\n if \"missing\" in page:\n print(f\"Error: Page '{WIKI_PAGE}' not found.\", file=sys.stderr)\n sys.exit(2)\n\n revision = page[\"revisions\"][0]\n rev_id = revision[\"revid\"]\n timestamp = revision[\"timestamp\"]\n content = revision[\"slots\"][\"main\"][\"content\"]\n return rev_id, timestamp, content\n\n\ndef get_wikitext(from_file: str | None) -> tuple[int, str, str]:\n \"\"\"Return (revision_id, timestamp, wikitext).\n\n With `from_file` set, read wikitext from that local path instead of\n hitting the network — used for offline eval/fixture runs. There is no\n real revision for a local file, so revision_id/timestamp are placeholders.\n \"\"\"\n if from_file:\n content = Path(from_file).read_text(encoding=\"utf-8\")\n return 0, \"from-file\", content\n return fetch_latest_revision()\n\n\ndef load_state() -> SyncState | None:\n if not STATE_FILE.exists():\n return None\n with open(STATE_FILE, \"r\") as f:\n return json.load(f)\n\n\ndef save_state(rev_id: int, timestamp: str, content: str) -> None:\n state: SyncState = {\n \"last_revision_id\": rev_id,\n \"last_timestamp\": timestamp,\n \"content_hash\": hashlib.sha256(content.encode()).hexdigest(),\n \"last_content_snapshot\": content,\n }\n with open(STATE_FILE, \"w\") as f:\n json.dump(state, f, indent=2)\n\n\ndef parse_wikitext(wikitext: str) -> list[ParsedSection]:\n \"\"\"Parse wikitext into structured sections with watch words and examples.\"\"\"\n sections: list[ParsedSection] = []\n current_title = \"Introduction\"\n current_level = 1\n current_lines: list[str] = []\n\n for line in wikitext.split(\"\\n\"):\n header_match = re.match(r\"^(={2,6})\\s*(.+?)\\s*\\1\\s*$\", line)\n if header_match:\n if current_lines:\n sections.append(_build_section(current_title, current_level, current_lines))\n current_level = len(header_match.group(1))\n current_title = header_match.group(2).strip()\n current_lines = []\n else:\n current_lines.append(line)\n\n if current_lines:\n sections.append(_build_section(current_title, current_level, current_lines))\n\n return sections\n\n\ndef _build_section(title: str, level: int, lines: list[str]) -> ParsedSection:\n \"\"\"Build a ParsedSection from raw lines.\"\"\"\n content = \"\\n\".join(lines).strip()\n watch_words = _extract_watch_words(content)\n examples = _extract_examples(content)\n return {\n \"title\": title,\n \"level\": level,\n \"content\": content,\n \"watch_words\": watch_words,\n \"examples\": examples,\n }\n\n\ndef _extract_watch_words(content: str) -> list[str]:\n \"\"\"Extract words-to-watch from tmbox templates and bold/italic markers.\"\"\"\n words: list[str] = []\n\n # Match {{tmbox|text=...}} content\n for match in re.finditer(r\"\\{\\{tmbox\\|[^}]*text\\s*=\\s*([^}]+)\\}\\}\", content, re.DOTALL):\n text = match.group(1)\n # Extract bold words within tmbox\n for bold in re.finditer(r\"'''(.+?)'''\", text):\n words.append(bold.group(1).strip())\n\n # Match bold words in \"words to watch\" style lists\n for match in re.finditer(r\"'''(.+?)'''\", content):\n word = match.group(1).strip()\n if word and len(word) \u003c 50:\n words.append(word)\n\n # Match items in bulleted lists\n for match in re.finditer(r\"^\\*\\s*'''(.+?)'''\", content, re.MULTILINE):\n words.append(match.group(1).strip())\n\n return list(dict.fromkeys(words)) # dedupe preserving order\n\n\ndef _extract_examples(content: str) -> list[str]:\n \"\"\"Extract example blocks (blockquotes, textdiff templates).\"\"\"\n examples: list[str] = []\n\n # {{blockquote|...}}\n for match in re.finditer(r\"\\{\\{blockquote\\|([^}]+)\\}\\}\", content, re.DOTALL):\n examples.append(match.group(1).strip())\n\n # {{textdiff|old=...|new=...}}\n for match in re.finditer(\n r\"\\{\\{textdiff\\|[^}]*old\\s*=\\s*([^|]+)\\|[^}]*new\\s*=\\s*([^}]+)\\}\\}\",\n content, re.DOTALL\n ):\n examples.append(f\"BEFORE: {match.group(1).strip()}\\nAFTER: {match.group(2).strip()}\")\n\n # Indented blockquote lines (: prefix in wikitext)\n block_lines: list[str] = []\n for line in content.split(\"\\n\"):\n if line.startswith(\":\"):\n block_lines.append(line.lstrip(\": \"))\n elif block_lines:\n examples.append(\"\\n\".join(block_lines))\n block_lines = []\n if block_lines:\n examples.append(\"\\n\".join(block_lines))\n\n return examples\n\n\ndef compute_diff(old_sections: list[ParsedSection], new_sections: list[ParsedSection]) -> list[Change]:\n \"\"\"Compute structured diff between two parsed page versions.\"\"\"\n changes: list[Change] = []\n\n old_by_title = {s[\"title\"]: s for s in old_sections}\n new_by_title = {s[\"title\"]: s for s in new_sections}\n\n for title, new_sec in new_by_title.items():\n if title not in old_by_title:\n changes.append({\n \"type\": \"new_section\",\n \"section\": title,\n \"details\": f\"New section with {len(new_sec['watch_words'])} watch words\",\n \"words\": new_sec[\"watch_words\"],\n })\n else:\n old_sec = old_by_title[title]\n new_words = set(new_sec[\"watch_words\"]) - set(old_sec[\"watch_words\"])\n if new_words:\n changes.append({\n \"type\": \"new_words\",\n \"section\": title,\n \"details\": f\"{len(new_words)} new watch words added\",\n \"words\": sorted(new_words),\n })\n\n old_hash = hashlib.sha256(old_sec[\"content\"].encode()).hexdigest()\n new_hash = hashlib.sha256(new_sec[\"content\"].encode()).hexdigest()\n if old_hash != new_hash and not new_words:\n changes.append({\n \"type\": \"modified\",\n \"section\": title,\n \"details\": \"Section content changed (no new watch words)\",\n \"words\": [],\n })\n\n for title in old_by_title:\n if title not in new_by_title:\n changes.append({\n \"type\": \"removed\",\n \"section\": title,\n \"details\": \"Section removed from Wikipedia page\",\n \"words\": old_by_title[title][\"watch_words\"],\n })\n\n return changes\n\n\n# Mapping from Wikipedia section themes to unslop target files/sections\nSECTION_MAP: dict[str, dict[str, str]] = {\n \"words to watch\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"AI Vocabulary (Additional)\",\n },\n \"promotional\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Promotional Language\",\n },\n \"significance\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Significance & Legacy Inflation\",\n },\n \"legacy\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Significance & Legacy Inflation\",\n },\n \"vague\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Vague Attributions\",\n },\n \"attribution\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Vague Attributions\",\n },\n \"copula\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Copula Avoidance\",\n },\n \"participial\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Superficial -ing Analyses\",\n },\n \"chatbot\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Communication Artifacts\",\n },\n \"jargon\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Business Jargon\",\n },\n \"structure\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Structural Patterns to Avoid\",\n },\n \"synonym\": {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"Elegant Variation / Synonym Cycling\",\n },\n}\n\n\ndef map_change_to_target(change: Change) -> dict[str, str]:\n \"\"\"Map a Wikipedia change to the appropriate unslop file and section.\"\"\"\n section_lower = change[\"section\"].lower()\n for keyword, target in SECTION_MAP.items():\n if keyword in section_lower:\n return target\n return {\n \"file\": \"references/taboo-phrases.md\",\n \"section\": \"AI Vocabulary (Additional)\",\n }\n\n\ndef generate_prompt(changes: list[Change], sections: list[ParsedSection]) -> str:\n \"\"\"Generate a Claude Code integration prompt from detected changes.\"\"\"\n if not changes:\n return \"No changes detected since last sync. Nothing to integrate.\"\n\n lines = [\n \"# Wikipedia AI Writing Patterns — Integration Update\",\n \"\",\n \"The Wikipedia 'Signs of AI writing' page has been updated.\",\n \"Below are changes that need to be integrated into the unslop skill.\",\n \"\",\n \"## Changes Detected\",\n \"\",\n ]\n\n for i, change in enumerate(changes, 1):\n target = map_change_to_target(change)\n lines.append(f\"### {i}. [{change['type'].upper()}] {change['section']}\")\n lines.append(f\"\")\n lines.append(f\"**Details:** {change['details']}\")\n lines.append(f\"**Target file:** `{target['file']}`\")\n lines.append(f\"**Target section:** {target['section']}\")\n if change[\"words\"]:\n lines.append(f\"**Words/phrases to add:**\")\n for word in change[\"words\"]:\n lines.append(f\"- \\\"{word}\\\"\")\n lines.append(\"\")\n\n lines.extend([\n \"## Integration Instructions\",\n \"\",\n \"For each change above:\",\n \"\",\n \"1. Read the target file\",\n \"2. Check if the phrase already exists (avoid duplicates)\",\n \"3. Add new phrases to the appropriate section in `references/taboo-phrases.md`\",\n \"4. Mirror additions in `scripts/banned_phrase_scan.py` BANNED_PHRASES dict\",\n \"5. If a new before/after example is warranted, add to `references/edit-library.md`\",\n \"\",\n \"After all changes:\",\n \"- Run `python3 scripts/banned_phrase_scan.py \u003c /dev/null` to verify no syntax errors\",\n \"- Verify no duplicate entries in taboo-phrases.md\",\n ])\n\n return \"\\n\".join(lines)\n\n\ndef cmd_check(from_file: str | None = None) -> None:\n \"\"\"Check for updates. Exit 0 = no updates, 1 = updates available.\"\"\"\n rev_id, timestamp, content = get_wikitext(from_file)\n state = load_state()\n\n content_hash = hashlib.sha256(content.encode()).hexdigest()\n\n if state is None:\n print(f\"No previous sync state. Current revision: {rev_id} ({timestamp})\")\n print(\"Run 'diff' or 'prompt' to see all content as new.\")\n sys.exit(1)\n\n if content_hash == state[\"content_hash\"]:\n print(f\"No changes. Current revision: {rev_id} ({timestamp})\")\n print(f\"Last synced revision: {state['last_revision_id']} ({state['last_timestamp']})\")\n sys.exit(0)\n\n print(f\"Updates available!\")\n print(f\" Last synced: revision {state['last_revision_id']} ({state['last_timestamp']})\")\n print(f\" Current: revision {rev_id} ({timestamp})\")\n sys.exit(1)\n\n\ndef cmd_diff(from_file: str | None = None) -> None:\n \"\"\"Output structured JSON diff of changes.\"\"\"\n rev_id, timestamp, content = get_wikitext(from_file)\n state = load_state()\n\n new_sections = parse_wikitext(content)\n\n if state is None:\n old_sections: list[ParsedSection] = []\n else:\n old_sections = parse_wikitext(state[\"last_content_snapshot\"])\n\n changes = compute_diff(old_sections, new_sections)\n\n output = {\n \"revision_id\": rev_id,\n \"timestamp\": timestamp,\n \"previous_revision_id\": state[\"last_revision_id\"] if state else None,\n \"total_changes\": len(changes),\n \"changes\": changes,\n }\n\n print(json.dumps(output, indent=2))\n if not from_file:\n save_state(rev_id, timestamp, content)\n\n\ndef cmd_prompt(from_file: str | None = None) -> None:\n \"\"\"Output a Claude Code integration prompt.\"\"\"\n rev_id, timestamp, content = get_wikitext(from_file)\n state = load_state()\n\n new_sections = parse_wikitext(content)\n\n if state is None:\n old_sections: list[ParsedSection] = []\n else:\n old_sections = parse_wikitext(state[\"last_content_snapshot\"])\n\n changes = compute_diff(old_sections, new_sections)\n prompt = generate_prompt(changes, new_sections)\n\n print(prompt)\n if not from_file:\n save_state(rev_id, timestamp, content)\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(\n description=\"Sync unslop rules with Wikipedia's 'Signs of AI writing' page.\"\n )\n subparsers = parser.add_subparsers(dest=\"command\")\n\n commands = {\n \"check\": cmd_check,\n \"diff\": cmd_diff,\n \"prompt\": cmd_prompt,\n }\n for name in commands:\n sub = subparsers.add_parser(name)\n sub.add_argument(\n \"--from-file\",\n metavar=\"PATH\",\n help=(\n \"read wikitext from PATH instead of fetching from Wikipedia \"\n \"(skips the network call and skips writing sync state)\"\n ),\n )\n\n if len(sys.argv) \u003c 2:\n print(\"Usage: wiki_sync.py \u003ccheck|diff|prompt>\", file=sys.stderr)\n sys.exit(2)\n\n if sys.argv[1] not in commands:\n print(f\"Unknown command: {sys.argv[1]}\", file=sys.stderr)\n print(f\"Available: {', '.join(commands)}\", file=sys.stderr)\n sys.exit(2)\n\n args = parser.parse_args()\n commands[args.command](args.from_file)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":15624,"content_sha256":"616b9c491a5c98c37a32678a6bbf051dfd909552fa921aa260bfd8acc80824a0"}],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Unslop","type":"text"}]},{"type":"paragraph","content":[{"text":"Humanize AI-generated prose. Audit first. Rewrite only when the user asks for a rewrite.","type":"text"}]},{"type":"paragraph","content":[{"text":"For every audit or rewrite, read ","type":"text"},{"text":"references/core-contract.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/core-contract.md","title":null}}]},{"text":". It is the single behavior contract. Command files define routing and mechanics; presets supply optional voice, but neither can override the core contract.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Routing","type":"text"}]},{"type":"paragraph","content":[{"text":"When the user invokes a sub-command (","type":"text","marks":[{"type":"strong"}]},{"text":"/unslop teach ...","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":", ","type":"text","marks":[{"type":"strong"}]},{"text":"/unslop cleanup ...","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":"), you MUST read ","type":"text","marks":[{"type":"strong"}]},{"text":"references/commands/\u003ccommand>.md","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" before acting. Non-optional — the command file defines the flow, and skipping it drops steps the user expects.","type":"text","marks":[{"type":"strong"}]},{"text":" A bare ","type":"text"},{"text":"/unslop \u003ctext>","type":"text","marks":[{"type":"code_inline"}]},{"text":" with no leading command word defaults to ","type":"text"},{"text":"rewrite","type":"text","marks":[{"type":"code_inline"}]},{"text":". If the first word does not match a command but the intent clearly maps to one (e.g. \"flag the AI tells, don't change anything\" → ","type":"text"},{"text":"cleanup","type":"text","marks":[{"type":"code_inline"}]},{"text":" report-only), load that command file and proceed as if invoked.","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Command","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Purpose","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"File","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"rewrite","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Default two-pass de-slop: diagnose, reconstruct under the guards, validate.","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/rewrite.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/rewrite.md","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"cleanup","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Co-writer: cheap detection, reviewable suggestions with contract gates; includes report-only \"flag, change nothing\".","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/cleanup.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/cleanup.md","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"teach","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Agent-driven voice building: harvest, approve, profile, layered card, scored demo.","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/teach.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/teach.md","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"mimic","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Voiced drafting or rewriting under the full gates; refine loop when one pass falls short.","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/mimic.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/mimic.md","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"maintenance","type":"text","marks":[{"type":"em"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Turn a wild AI-ism into an eval row and a PR (not a top-level verb).","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/contribute.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/contribute.md","title":null}}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Routing by phrase","type":"text"}]},{"type":"paragraph","content":[{"text":"Sub-flows are reachable by their natural names without being top-level verbs. When the user says any of these, load the named file and jump to the flow:","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"The user says","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Go to","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"audit","type":"text","marks":[{"type":"code_inline"}]},{"text":" / \"just flag it\" / \"don't change anything\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/cleanup.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/cleanup.md#report-only","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"review","type":"text","marks":[{"type":"code_inline"}]},{"text":" / \"review this before I publish\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/cleanup.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/cleanup.md#report-only","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"harvest","type":"text","marks":[{"type":"code_inline"}]},{"text":" / \"what writing of mine do you have?\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/teach.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/teach.md#1-gather-samples-harvest","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"calibrate","type":"text","marks":[{"type":"code_inline"}]},{"text":" / \"the A/B game\" / \"quiz me on my voice\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/teach.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/teach.md#calibrate","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"refine","type":"text","marks":[{"type":"code_inline"}]},{"text":" / \"keep pushing until it sounds like me\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/mimic.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/mimic.md#refine","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"voice check / \"does this sound like me?\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/mimic.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/mimic.md#voice-check","title":null}}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"\"found a new AI-ism\" / \"add this tell\"","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/contribute.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/commands/contribute.md","title":null}}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Interface","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Argument","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Default","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--preset","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Voice style: ","type":"text"},{"text":"crisp","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"warm","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"expert","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"story","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"crisp","type":"text","marks":[{"type":"code_inline"}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--strict","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Fail if rubric score \u003c 32/40","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"false","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--report","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Flag AI patterns without changing the text (cleanup)","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"false","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Input","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Text to transform (argument, file path, or stdin)","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"required","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"Read one preset from ","type":"text"},{"text":"presets/","type":"text","marks":[{"type":"code_inline"}]},{"text":" before writing.","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Preset","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Style","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Best For","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"crisp","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Short, direct, no fluff","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Technical writing, documentation","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"warm","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Friendly, conversational","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Emails, blog posts","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"expert","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Authoritative, confident","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Thought leadership, articles","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"story","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Narrative flow, show don't tell","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Case studies, personal posts","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"Rewrite, preservation, register, and validation behavior lives only in ","type":"text"},{"text":"references/core-contract.md","type":"text","marks":[{"type":"code_inline"}]},{"text":"; do not recreate or override those rules here.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Output Format","type":"text"}]},{"type":"paragraph","content":[{"text":"For a quick rewrite, return the cleaned text only. For audit-only (cleanup ","type":"text"},{"text":"--report","type":"text","marks":[{"type":"code_inline"}]},{"text":"):","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"markdown"},"content":[{"text":"## Issues Found\n\n- [Quoted issue, category, severity, why it reads as AI]\n\n## Assessment\n\n- [Which issues are clear problems]\n- [Which issues are judgment calls or context-dependent]","type":"text"}]},{"type":"paragraph","content":[{"text":"For strict or requested analysis:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"markdown"},"content":[{"text":"## Transformed Text\n\n[The humanized version]\n\n## Validation\n\n- Constraints: [X]/[Y] preserved\n- AI patterns: [N] remaining (was [M])\n- Structure: [pass/fail]\n- Readability: Grade [X], sentence variance [Y]\n- Change: [X]% from original\n- Score: [X]/40","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Reference Files","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"File","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"When to Read","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/commands/*.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"The routed command flows (rewrite, cleanup, teach, mimic, contribute).","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/pipeline.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Orchestrated tiered execution for multi-agent harnesses.","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/taboo-phrases.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Authoritative phrase catalog and scanner categories.","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/fact-preservation.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Constraint preservation rules.","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/rewrite-examples.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Executable before/after examples.","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/{mimic,harvest,calibrate}.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Voice-tool internals loaded by their routed command.","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"references/{rubric,edit-library,maintenance}.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Strict scoring, examples, and contribution procedures.","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"presets/*.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Voice-specific deltas.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Maintenance","type":"text"}]},{"type":"paragraph","content":[{"text":"The eval contracts define the product. Add scanner examples eval-first in ","type":"text"},{"text":"evals/fixtures/contracts/scanner-examples.json","type":"text","marks":[{"type":"code_inline"}]},{"text":"; use ","type":"text"},{"text":"evals/adversarial-evals.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" for agent behavior and routing. Do not edit legacy ","type":"text"},{"text":"evals/evals.json","type":"text","marks":[{"type":"code_inline"}]},{"text":". New patterns need a false-negative example and a false-positive protection example. Agent behavior changes need a ","type":"text"},{"text":"skill","type":"text","marks":[{"type":"code_inline"}]},{"text":" row and a regenerated shared benchmark. For the concrete procedures (add a phrase or structure, list current patterns, sync with Wikipedia's signs-of-AI-writing page), read ","type":"text"},{"text":"references/maintenance.md","type":"text","marks":[{"type":"code_inline"}]},{"text":". Found a new AI-ism in the wild? ","type":"text"},{"text":"references/commands/contribute.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" turns the exact snippet into a contract example and a structured PR, keeping both user-confirmation gates.","type":"text"}]}]},"metadata":{"date":"2026-11-02","name":"unslop","author":"@skillopedia","source":{"stars":712,"repo_name":"unslop","origin_url":"https://github.com/theclaymethod/unslop/blob/HEAD/SKILL.md","repo_owner":"theclaymethod","body_sha256":"e71f7e90759226fcc75cf6c62feb110ec3a108f67edaa9f23f1396709620163b","cluster_key":"ec6d14c4c29111412e3be415538a613a9df049ec2119597300421a32a268c5ec","clean_bundle":{"format":"clean-skill-bundle-v1","source":"theclaymethod/unslop/SKILL.md","text_bytes":452142,"source_commit":"17ed39c9d0b522f44190ff0c6233867eadee192a","skills_sh_rank":4663,"attachment_count":51,"text_attachments":51,"attachment_storage":"skill_attachments","binary_attachments":0},"cluster_size":1,"skill_md_path":"SKILL.md","import_metadata":{"date":"2026-11-02","author":"@skillopedia","version":"v1","category":"documents-office","category_label":"Documents"},"exact_dupes_collapsed_into_this":0},"license":"MIT","version":"v1","category":"documents-office","metadata":{"author":"claytonkim","version":"2.3.0"},"import_tag":"skills-sh-top20k-2026-11","description":"Remove AI writing patterns from prose using either audit-only detection or a two-pass rewrite flow (diagnosis then reconstruction). Use this skill when editing, reviewing, or rewriting AI-generated content to make it sound human. Triggers on requests to \"humanize\", \"de-slop\", \"fix AI text\", \"make it sound human\", \"remove AI patterns\", or when reviewing text that contains obvious AI tells like \"Here's the thing:\", \"Let that sink in\", or \"In today's fast-paced landscape\". Also use when the user pastes text and says it \"sounds like ChatGPT\", \"sounds robotic\", \"needs to sound more natural\", or asks you to \"clean up\" drafted content before publishing.","argument-hint":"[teach · cleanup · rewrite · mimic] [input]","user-invocable":true}},"renderedAt":1793666256076}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.