translate-book — Skillopedia

Book Translation Skill You are a book translation assistant. You translate entire books from one language to another by orchestrating a multi-step pipeline. Workflow 1. Collect Parameters Determine the following from the user's message: - file path : Path to the input file (PDF, DOCX, EPUB, or Markdown) — REQUIRED - target lang : Target language code (default: ) — e.g. zh, en, ja, ko, fr, de, es - concurrency : Number of parallel sub-agents per batch (default: ) - temp root : Optional directory under which should be created - epub cover : Optional explicit cover image path for EPUB output - e…

)\n\n\ndef parse_chunk_name(chunk_name):\n \"\"\"Return (chunk_id, number, width) for chunkNNNN.md.\"\"\"\n base = os.path.basename(chunk_name)\n match = CHUNK_RE.match(base)\n if not match:\n raise ValueError(f\"Expected chunk filename like chunk0001.md, got {base!r}\")\n digits = match.group(1)\n return f\"chunk{digits}\", int(digits), len(digits)\n\n\ndef _neighbor_path(temp_dir, number, width):\n if number \u003c 1:\n return None\n path = Path(temp_dir) / f\"chunk{number:0{width}d}.md\"\n return path if path.exists() else None\n\n\ndef _read_excerpt(path, chars, tail=False):\n if path is None:\n return \"\"\n text = path.read_text(encoding='utf-8')\n excerpt = text[-chars:] if tail else text[:chars]\n return excerpt.strip()\n\n\ndef get_neighbor_context(temp_dir, chunk_name, chars=300):\n \"\"\"Return neighbor context dict for a chunk.\n\n `prev_excerpt` is the tail of the previous source chunk; `next_excerpt` is\n the head of the next source chunk. Missing neighbors produce empty strings.\n \"\"\"\n if chars \u003c 0:\n raise ValueError(\"chars must be non-negative\")\n\n chunk_id, number, width = parse_chunk_name(chunk_name)\n temp_path = Path(temp_dir)\n source_path = temp_path / os.path.basename(chunk_name)\n if not source_path.exists():\n raise FileNotFoundError(f\"Source chunk not found: {source_path}\")\n\n prev_path = _neighbor_path(temp_path, number - 1, width)\n next_path = _neighbor_path(temp_path, number + 1, width)\n\n return {\n \"chunk_id\": chunk_id,\n \"prev_chunk\": prev_path.name if prev_path else None,\n \"next_chunk\": next_path.name if next_path else None,\n \"prev_excerpt\": _read_excerpt(prev_path, chars, tail=True),\n \"next_excerpt\": _read_excerpt(next_path, chars, tail=False),\n }\n\n\ndef format_for_prompt(context):\n \"\"\"Render a concise prompt block. Empty context renders to empty string.\"\"\"\n parts = []\n if context.get(\"prev_excerpt\"):\n parts.append(\n f\"Previous chunk excerpt ({context['prev_chunk']}, read-only):\\n\"\n f\"```text\\n{context['prev_excerpt']}\\n```\"\n )\n if context.get(\"next_excerpt\"):\n parts.append(\n f\"Next chunk excerpt ({context['next_chunk']}, read-only):\\n\"\n f\"```text\\n{context['next_excerpt']}\\n```\"\n )\n return \"\\n\\n\".join(parts)\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Print read-only neighboring chunk excerpts for a translation prompt\"\n )\n parser.add_argument(\"temp_dir\", help=\"Path to \u003cbook>_temp/ directory\")\n parser.add_argument(\"chunk_file\", help=\"Chunk filename such as chunk0001.md\")\n parser.add_argument(\n \"--chars\",\n type=int,\n default=300,\n help=\"Number of characters to take from each neighbor (default: 300)\",\n )\n parser.add_argument(\n \"--json\",\n action=\"store_true\",\n help=\"Emit JSON instead of a prompt-ready markdown block\",\n )\n args = parser.parse_args()\n\n try:\n context = get_neighbor_context(args.temp_dir, args.chunk_file, args.chars)\n except Exception as e:\n print(f\"Error: {e}\", file=sys.stderr)\n sys.exit(1)\n\n if args.json:\n print(json.dumps(context, ensure_ascii=False, indent=2))\n else:\n block = format_for_prompt(context)\n if block:\n print(block)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":3897,"content_sha256":"b867284e4311f8a6a16786e763e4519c5c3391380f0f37c73475848cac8e8d25"},{"filename":"scripts/convert.py","content":"#!/usr/bin/env python3\n\"\"\"\nconvert.py - Convert PDF/DOCX/EPUB to Markdown chunks via Calibre HTMLZ\nCombines the original steps 1-2 into a single script.\n\nMarkdown input (.md/.markdown) skips Calibre entirely. Use it for PDFs that\nCalibre's coordinate-heuristic reflow mangles (formulas, tables, multi-column\nlayouts): extract them with a layout-aware parser such as MinerU or Marker\nfirst, then pass the resulting Markdown here.\n\"\"\"\n\nimport os\nimport sys\nimport subprocess\nimport zipfile\nimport shutil\nimport tempfile\nimport argparse\nimport base64\nimport binascii\nimport bisect\nimport glob\nimport hashlib\nimport json\nimport re\nimport urllib.parse\n\nfrom manifest import create_manifest, file_hash\n\n\ndef find_calibre_convert():\n \"\"\"Find ebook-convert command from Calibre installation\"\"\"\n possible_paths = [\n \"/Applications/calibre.app/Contents/MacOS/ebook-convert\",\n \"/usr/bin/ebook-convert\",\n \"/usr/local/bin/ebook-convert\",\n \"ebook-convert\" # If in PATH\n ]\n\n for path in possible_paths:\n try:\n result = subprocess.run([path, \"--version\"], capture_output=True, text=True, timeout=10)\n if result.returncode == 0:\n print(f\"Found Calibre ebook-convert: {path}\")\n return path\n except (FileNotFoundError, subprocess.TimeoutExpired):\n continue\n\n return None\n\n\ndef convert_to_htmlz(input_file, htmlz_file, calibre_path):\n \"\"\"Convert input file to HTMLZ using Calibre\"\"\"\n try:\n print(f\"Converting {input_file} to HTMLZ...\")\n cmd = [calibre_path, input_file, htmlz_file]\n result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)\n\n if result.returncode == 0:\n file_size = os.path.getsize(htmlz_file)\n print(f\"HTMLZ conversion successful: {htmlz_file} ({file_size} bytes)\")\n return True\n else:\n print(f\"HTMLZ conversion failed: {result.stderr}\")\n return False\n except subprocess.TimeoutExpired:\n print(\"HTMLZ conversion timed out\")\n return False\n except Exception as e:\n print(f\"HTMLZ conversion error: {e}\")\n return False\n\n\ndef extract_metadata_from_htmlz(extract_dir):\n \"\"\"Extract metadata from metadata.opf file in HTMLZ\"\"\"\n try:\n import xml.etree.ElementTree as ET\n\n metadata_file = None\n for root, dirs, files in os.walk(extract_dir):\n for file in files:\n if file.lower() == 'metadata.opf':\n metadata_file = os.path.join(root, file)\n break\n if metadata_file:\n break\n\n if not metadata_file:\n return {}\n\n tree = ET.parse(metadata_file)\n root = tree.getroot()\n\n namespaces = {\n 'opf': 'http://www.idpf.org/2007/opf',\n 'dc': 'http://purl.org/dc/elements/1.1/',\n 'dcterms': 'http://purl.org/dc/terms/'\n }\n\n metadata = {}\n\n title_elem = root.find('.//dc:title', namespaces)\n if title_elem is not None and title_elem.text:\n metadata['title'] = title_elem.text.strip()\n\n creator_elem = root.find('.//dc:creator', namespaces)\n if creator_elem is not None and creator_elem.text:\n metadata['creator'] = creator_elem.text.strip()\n\n publisher_elem = root.find('.//dc:publisher', namespaces)\n if publisher_elem is not None and publisher_elem.text:\n metadata['publisher'] = publisher_elem.text.strip()\n\n language_elem = root.find('.//dc:language', namespaces)\n if language_elem is not None and language_elem.text:\n metadata['language'] = language_elem.text.strip()\n\n return metadata\n\n except Exception as e:\n print(f\"Warning: Error extracting metadata: {e}\")\n return {}\n\n\ndef extract_htmlz(htmlz_file, temp_dir):\n \"\"\"Extract HTMLZ file and return paths to HTML and images\"\"\"\n try:\n with zipfile.ZipFile(htmlz_file, 'r') as zip_file:\n zip_file.extractall(temp_dir)\n\n html_file = None\n images_dir = None\n\n for root, dirs, files in os.walk(temp_dir):\n for file in files:\n if file.lower() in ['index.html', 'index.htm']:\n html_file = os.path.join(root, file)\n break\n for dir_name in dirs:\n if dir_name.lower() in ['images', 'image', 'pics', 'pictures']:\n images_dir = os.path.join(root, dir_name)\n break\n\n if not html_file:\n for root, dirs, files in os.walk(temp_dir):\n for file in files:\n if file.lower().endswith(('.html', '.htm')):\n html_file = os.path.join(root, file)\n break\n if html_file:\n break\n\n return html_file, images_dir\n\n except Exception as e:\n print(f\"Error extracting HTMLZ: {e}\")\n return None, None\n\n\ndef build_temp_dir(input_file, temp_root=None):\n \"\"\"Return the working directory path for an input file.\n\n Default is the historical cwd-local {book_name}_temp/. When temp_root is\n provided, only the root changes; the leaf directory name stays compatible.\n \"\"\"\n base_name = os.path.splitext(os.path.basename(input_file))[0]\n leaf = f\"{base_name}_temp\"\n if temp_root:\n return os.path.join(temp_root, leaf)\n return leaf\n\n\nSOURCE_FINGERPRINT_FILE = \"source_fingerprint.json\"\n\n# Files whose presence means the temp dir carries conversion state derived\n# from some source book — and must therefore be tied to the current one.\n_SOURCE_CACHE_MARKERS = (\n \"input.html\",\n \"input.md\",\n \"manifest.json\",\n \"run_state.json\",\n \"glossary.json\",\n \"output.md\",\n)\n\n\ndef source_fingerprint(input_file):\n \"\"\"Stable identity of the exact source bytes being converted.\"\"\"\n return {\n \"path\": os.path.realpath(input_file),\n \"size\": os.path.getsize(input_file),\n \"sha256\": file_hash(input_file),\n }\n\n\ndef _write_source_fingerprint(temp_dir, fingerprint):\n os.makedirs(temp_dir, exist_ok=True)\n path = os.path.join(temp_dir, SOURCE_FINGERPRINT_FILE)\n with open(path, 'w', encoding='utf-8') as f:\n json.dump(fingerprint, f, indent=2, sort_keys=True)\n f.write('\\n')\n\n\ndef _load_source_fingerprint(temp_dir):\n path = os.path.join(temp_dir, SOURCE_FINGERPRINT_FILE)\n if not os.path.exists(path):\n return None\n try:\n with open(path, 'r', encoding='utf-8') as f:\n return json.load(f)\n except (OSError, json.JSONDecodeError):\n return None\n\n\ndef _has_reusable_source_cache(temp_dir):\n if not os.path.isdir(temp_dir):\n return False\n for name in _SOURCE_CACHE_MARKERS:\n if os.path.exists(os.path.join(temp_dir, name)):\n return True\n for pattern in ('chunk*.md', 'output_chunk*.md'):\n if glob.glob(os.path.join(temp_dir, pattern)):\n return True\n return False\n\n\ndef check_source_cache(temp_dir, current_fingerprint):\n \"\"\"Compare the temp dir's cached source identity against the current input.\n\n Returns (status, message):\n (None, None) — fresh temp dir, or fingerprint matches; proceed.\n ('adopt', message) — cache predates fingerprinting; adopt it and record\n the current fingerprint (trust-on-first-use, keeps\n pre-upgrade temp dirs resumable).\n ('mismatch', message) — cache was built from different source bytes;\n the caller must abort.\n\n Only content identity (sha256 + size) is compared — moving or renaming the\n source file does not invalidate the cache.\n \"\"\"\n if not _has_reusable_source_cache(temp_dir):\n return None, None\n\n stored = _load_source_fingerprint(temp_dir)\n if stored is None:\n return 'adopt', (\n f\"{temp_dir}/ contains cached conversion artifacts without a \"\n f\"{SOURCE_FINGERPRINT_FILE} (created by an older version). \"\n f\"Assuming they were built from the current input file. \"\n f\"If you replaced the source file, delete {temp_dir}/ and re-run.\"\n )\n\n for key in (\"sha256\", \"size\"):\n if stored.get(key) != current_fingerprint.get(key):\n return 'mismatch', (\n f\"{temp_dir}/ was created from different source bytes \"\n f\"(cached sha256 {str(stored.get('sha256', ''))[:12]}..., \"\n f\"current {current_fingerprint['sha256'][:12]}...). \"\n f\"Reusing its chunks would translate the wrong book.\"\n )\n return None, None\n\n\ndef _abort_on_source_cache_mismatch(status, message, temp_dir):\n if status == 'mismatch':\n print(f\"Error: {message}\")\n print(f\"Delete {temp_dir}/ (or use a fresh --temp-root) and re-run.\")\n sys.exit(1)\n if status == 'adopt':\n print(f\"Warning: {message}\")\n\n\ndef setup_temp_directory(input_file, html_file, images_dir, temp_root=None):\n \"\"\"Setup temp directory with HTML and images\"\"\"\n try:\n temp_dir = build_temp_dir(input_file, temp_root)\n os.makedirs(temp_dir, exist_ok=True)\n\n input_html = os.path.join(temp_dir, \"input.html\")\n if os.path.exists(input_html):\n print(f\"Skipping HTML copy - input.html already exists\")\n else:\n shutil.copy2(html_file, input_html)\n print(f\"Copied HTML to: {input_html}\")\n\n if images_dir and os.path.exists(images_dir):\n target_images_dir = os.path.join(temp_dir, \"images\")\n if os.path.exists(target_images_dir):\n print(f\"Skipping images copy - images directory already exists\")\n else:\n shutil.copytree(images_dir, target_images_dir)\n print(f\"Copied images to: {target_images_dir}\")\n\n return temp_dir\n except Exception as e:\n print(f\"Error setting up temp directory: {e}\")\n return None\n\n\n# =============================================================================\n# Structural cleanup on pandoc's JSON AST (Calibre HTML -> Markdown)\n# =============================================================================\n\n_INLINE_TYPES = frozenset({\n \"Str\", \"Emph\", \"Underline\", \"Strong\", \"Strikeout\", \"Superscript\", \"Subscript\",\n \"SmallCaps\", \"Quoted\", \"Cite\", \"Code\", \"Space\", \"SoftBreak\", \"LineBreak\",\n \"Math\", \"RawInline\", \"Link\", \"Image\", \"Note\", \"Span\",\n})\n_SPACE_TYPES = frozenset({\"Space\", \"SoftBreak\", \"LineBreak\"})\n_FORMATTING_TYPES = frozenset({\n \"Emph\", \"Underline\", \"Strong\", \"Strikeout\", \"Superscript\", \"Subscript\", \"SmallCaps\",\n})\n# Digits or note symbols, optionally bracketed and/or followed by a period:\n# \"1\", \"[12]\", \"(3)\", \"4.\", \"*\", \"†\".\n_FOOTNOTE_MARKER_RE = re.compile(r'^[\\[(]?(\\d{1,4}|[*†‡§¶‖]{1,3})[\\])]?\\.?

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

)\n_NOTE_MARKER_TRAILERS = frozenset({\".\", \")\", \"]\", \":\"})\n\n\ndef _map_inline_lists(node, fn, top_down=False):\n \"\"\"Rewrite every inline list of a pandoc JSON AST in place via fn(list) -> list.\"\"\"\n if isinstance(node, dict):\n if isinstance(node.get(\"c\"), list):\n _map_inline_lists(node[\"c\"], fn, top_down)\n return\n if not isinstance(node, list):\n return\n is_inline_list = bool(node) and all(\n isinstance(x, dict) and x.get(\"t\") in _INLINE_TYPES for x in node\n )\n if is_inline_list and top_down:\n node[:] = fn(node)\n for item in node:\n _map_inline_lists(item, fn, top_down)\n if is_inline_list and not top_down:\n node[:] = fn(node)\n\n\ndef _inline_text(inlines):\n parts = []\n for x in inlines:\n t = x.get(\"t\")\n if t == \"Str\":\n parts.append(x[\"c\"])\n elif t in _SPACE_TYPES:\n parts.append(\" \")\n elif t in _FORMATTING_TYPES:\n parts.append(_inline_text(x[\"c\"]))\n elif t in (\"Span\", \"Link\", \"Quoted\"):\n parts.append(_inline_text(x[\"c\"][1]))\n elif t == \"Code\":\n parts.append(x[\"c\"][1])\n return \"\".join(parts)\n\n\ndef _is_anchor(inline):\n \"\"\"An element that renders nothing, e.g. an id-only \u003ca id=\"page_12\"/>.\"\"\"\n if inline.get(\"t\") not in (\"Span\", \"Link\"):\n return False\n return all(_is_anchor(x) for x in inline[\"c\"][1])\n\n\ndef _is_filler(inline):\n \"\"\"Whitespace, or a span/link holding nothing but whitespace and anchors.\"\"\"\n if inline.get(\"t\") in _SPACE_TYPES:\n return True\n if inline.get(\"t\") not in (\"Span\", \"Link\"):\n return False\n return all(_is_filler(x) for x in inline[\"c\"][1])\n\n\ndef _footnote_marker(inlines):\n match = _FOOTNOTE_MARKER_RE.match(_inline_text(inlines).strip())\n return match.group(1) if match else None\n\n\ndef _footnote_link(inline):\n \"\"\"Return (link, anchor ids) when inline is a footnote-marker link to an in-book target.\n\n The link may sit alone or inside \u003csup>/\u003cspan> wrappers; the ids are every\n anchor a note's back-link may point at (the link's own id and its wrapping\n spans' ids).\n \"\"\"\n t = inline.get(\"t\")\n if t == \"Link\":\n attr, content, (url, _title) = inline[\"c\"]\n if url.startswith(\"#\") and len(url) > 1 and _footnote_marker(content):\n return inline, [attr[0]] if attr[0] else []\n return None\n if t in (\"Superscript\", \"Span\"):\n content = inline[\"c\"] if t == \"Superscript\" else inline[\"c\"][1]\n solid = [x for x in content if not _is_filler(x)]\n if len(solid) == 1:\n found = _footnote_link(solid[0])\n if found and t == \"Span\" and inline[\"c\"][0][0]:\n found[1].append(inline[\"c\"][0][0])\n return found\n return None\n\n\ndef _index_footnote_refs(blocks):\n \"\"\"Map anchor id -> footnote reference link, for marker links inside running text.\n\n A marker link that opens its line is a note's back-link or a contents\n entry, never a reference, so it is not indexed.\n \"\"\"\n refs = {}\n\n def visit(inlines):\n after_text = False\n for x in inlines:\n if after_text:\n found = _footnote_link(x)\n if found:\n link, anchor_ids = found\n for anchor_id in anchor_ids:\n refs.setdefault(anchor_id, link)\n if not _is_filler(x):\n after_text = True\n return inlines\n\n _map_inline_lists(blocks, visit)\n return refs\n\n\ndef _note_body(block, refs):\n \"\"\"Return (reference link, note inlines) when block opens with a back-link to a known reference.\"\"\"\n if block.get(\"t\") not in (\"Para\", \"Plain\"):\n return None\n inlines = block[\"c\"]\n i = 0\n while i \u003c len(inlines) and _is_filler(inlines[i]):\n i += 1\n found = _footnote_link(inlines[i]) if i \u003c len(inlines) else None\n if not found:\n return None\n backlink = found[0]\n ref = refs.get(backlink[\"c\"][2][0][1:])\n if ref is None or _footnote_marker(ref[\"c\"][1]) != _footnote_marker(backlink[\"c\"][1]):\n return None\n body = inlines[i + 1:]\n while body and (\n _is_filler(body[0])\n or (body[0].get(\"t\") == \"Str\" and body[0][\"c\"] in _NOTE_MARKER_TRAILERS)\n ):\n body = body[1:]\n return (ref, body) if body else None\n\n\ndef _collect_notes(blocks, refs, found):\n \"\"\"Append (ref, container, block, note blocks) for note paragraphs in body/div block lists.\"\"\"\n for block in blocks:\n if block.get(\"t\") == \"Div\":\n attr, inner = block[\"c\"]\n solid = [b for b in inner if not (\n b.get(\"t\") in (\"Para\", \"Plain\") and all(_is_filler(x) for x in b[\"c\"])\n )]\n hit = _note_body(solid[0], refs) if solid else None\n # A div is the whole note only when the reference points at the div itself.\n if hit and attr[0] and hit[0][\"c\"][2][0] == \"#\" + attr[0]:\n found.append((hit[0], blocks, block, [{\"t\": \"Para\", \"c\": hit[1]}] + solid[1:]))\n continue\n _collect_notes(inner, refs, found)\n continue\n hit = _note_body(block, refs)\n if hit:\n found.append((hit[0], blocks, block, [{\"t\": \"Para\", \"c\": hit[1]}]))\n\n\ndef _pandoc_notes(node):\n \"\"\"Return the block lists of the Note elements found anywhere in node.\"\"\"\n found = []\n if isinstance(node, dict):\n if node.get(\"t\") == \"Note\":\n found.append(node[\"c\"])\n if isinstance(node.get(\"c\"), list):\n found.extend(_pandoc_notes(node[\"c\"]))\n elif isinstance(node, list):\n for item in node:\n found.extend(_pandoc_notes(item))\n return found\n\n\ndef convert_footnotes(blocks):\n \"\"\"Turn EPUB-style footnotes into pandoc notes. Returns the number converted.\n\n A reference is a short marker link in running text (`\u003ca href=\"#n1\">\u003csup>1\u003c/sup>\u003c/a>`,\n `\u003csup>\u003ca>1\u003c/a>\u003c/sup>`, `[1]`); its note is the paragraph (or the div the\n reference points at) that opens with a back-link to that reference with the\n same marker. Calibre renames ids and classes, so pairing relies only on this\n two-way link structure. Pandoc's Markdown writer then numbers the notes\n across the whole book, so labels stay unique when per-chapter numbering\n restarts.\n \"\"\"\n refs = _index_footnote_refs(blocks)\n if not refs:\n return 0\n candidates = []\n _collect_notes(blocks, refs, candidates)\n\n notes = {}\n for ref, container, block, content in candidates:\n notes.setdefault(id(ref), (container, block, content))\n # Pandoc before 3.13 drops a note written inside another note, so nothing\n # that would end up nested is converted: a reference inside a note (one\n # pandoc already read, e.g. a DPUB-ARIA note, or one converted here) stays\n # inline, and a note whose text holds a pandoc note stays a paragraph.\n nested = set()\n for note in _pandoc_notes(blocks) + [[block] for _container, block, _content in notes.values()]:\n nested.update(id(link) for link in _index_footnote_refs(note).values())\n for key, (_container, _block, content) in notes.items():\n if _pandoc_notes(content):\n nested.add(key)\n for key in nested:\n notes.pop(key, None)\n\n used = set()\n\n def replace(inlines):\n out = []\n for x in inlines:\n found = _footnote_link(x)\n key = id(found[0]) if found else None\n if key in notes and key not in used:\n used.add(key)\n out.append({\"t\": \"Note\", \"c\": notes[key][2]})\n else:\n out.append(x)\n return out\n\n _map_inline_lists(blocks, replace, top_down=True)\n\n for key in used:\n container, block, _content = notes[key]\n for i, candidate in enumerate(container):\n if candidate is block:\n del container[i]\n break\n return len(used)\n\n\ndef _unwrap_inline_markup(inlines):\n \"\"\"Unwrap spans and in-book links, clear the attributes of other links, and\n drop empty anchors inside text.\n\n Lists made only of anchors are left alone: they become `[]` lines that\n clean_calibre_markers() drops after its page-number pass, which keeps that\n pass seeing the same neighbouring lines as before.\n \"\"\"\n if all(_is_filler(x) for x in inlines):\n return inlines\n out = []\n # Unwrapped children are processed in turn, so an anchor nested in a span\n # is still dropped while the span's spaces and line breaks are kept.\n pending = list(reversed(inlines))\n while pending:\n x = pending.pop()\n t = x.get(\"t\")\n if _is_anchor(x):\n continue\n if t == \"Span\" or (t == \"Link\" and (x[\"c\"][2][0].startswith(\"#\") or _is_filler(x))):\n pending.extend(reversed(x[\"c\"][1]))\n else:\n if t == \"Link\":\n x[\"c\"][0] = [\"\", [], []]\n out.append(x)\n # Re-join text split by an unwrapped element, as pandoc's readers would:\n # the Markdown writer only escapes a leading \"4.\" or \"#\" it sees in one Str.\n merged = []\n for x in out:\n if merged and x.get(\"t\") == \"Str\" and merged[-1].get(\"t\") == \"Str\":\n merged[-1] = {\"t\": \"Str\", \"c\": merged[-1][\"c\"] + x[\"c\"]}\n else:\n merged.append(x)\n return merged\n\n\ndef _clear_header_attributes(node):\n if isinstance(node, dict):\n if node.get(\"t\") == \"Header\":\n node[\"c\"][1] = [\"\", [], []]\n if isinstance(node.get(\"c\"), list):\n _clear_header_attributes(node[\"c\"])\n elif isinstance(node, list):\n for item in node:\n _clear_header_attributes(item)\n\n\ndef clean_calibre_ast(doc):\n \"\"\"Structural cleanup of a pandoc JSON AST read from Calibre's HTML, in place.\n\n - EPUB footnotes become real pandoc notes (see convert_footnotes()).\n - In-book links lose their targets but keep their text: the ids they point\n at do not survive into the translated book.\n - Spans are unwrapped, so `{lang=\"…\"}` and class attributes never reach\n the Markdown.\n - Empty anchors inside text (`\u003ca id=\"page_12\"/>`) are dropped.\n - Heading and link attributes are cleared, whatever their id or class\n (external links keep their URL and title).\n \"\"\"\n blocks = doc.get(\"blocks\", [])\n footnotes = convert_footnotes(blocks)\n _map_inline_lists(blocks, _unwrap_inline_markup)\n _clear_header_attributes(blocks)\n return footnotes\n\n\ndef convert_html_to_markdown(html_file, md_file, strip_page_numbers=False, detect_page_numbers=True):\n \"\"\"Convert HTML to Markdown using pandoc\"\"\"\n try:\n import pypandoc\n\n doc = json.loads(pypandoc.convert_file(html_file, 'json'))\n footnotes = clean_calibre_ast(doc)\n if footnotes:\n print(f\"Converted {footnotes} footnote(s) to Markdown footnotes\")\n pypandoc.convert_text(\n json.dumps(doc),\n # Without smart, typographic dashes and quotes stay Unicode instead\n # of becoming ---, -- and backslash-escaped ASCII quotes.\n 'markdown-smart',\n format='json',\n outputfile=md_file,\n # block puts each note right after the paragraph citing it, so a\n # note and its reference usually land in the same chunk.\n extra_args=['--wrap=none', '--reference-location=block'],\n )\n\n if os.path.exists(md_file):\n with open(md_file, 'r', encoding='utf-8') as f:\n content = f.read()\n\n content = content.replace('\\ufeff', '')\n content = content.replace('\\u00a0', ' ')\n content = clean_calibre_markers(\n content,\n strip_page_numbers=strip_page_numbers,\n detect_page_numbers=detect_page_numbers,\n )\n\n with open(md_file, 'w', encoding='utf-8') as f:\n f.write(content)\n\n print(f\"Markdown conversion successful: {md_file}\")\n return True\n else:\n print(\"Markdown file was not created\")\n return False\n except ImportError:\n print(\"pypandoc not found. Install with: pip install pypandoc\")\n return False\n except Exception as e:\n print(f\"HTML to Markdown conversion failed: {e}\")\n return False\n\n\n_PAGE_SEQUENCE_MIN_LENGTH = 4\n_PAGE_SEQUENCE_MIN_RATIO = 0.5\n\n\ndef _detect_page_number_lines(lines):\n \"\"\"Detect standalone-digit lines that form a monotonic page-number sequence.\n\n Returns a set of line indices that should be dropped as page numbers.\n\n Algorithm: collect every standalone-digit line in document order, find the\n Longest Non-Decreasing Subsequence (LNDS) of their integer values via\n bisect_right with parent-pointer reconstruction. If the LNDS is long enough\n and covers a large enough fraction of all standalone digits, treat those\n elements as page numbers. Outliers (years like 1984, chapter numbers,\n citation indices) sit off the monotonic spine and stay preserved.\n \"\"\"\n digit_indices = []\n digit_values = []\n for i, line in enumerate(lines):\n s = line.strip()\n if s.isdigit():\n digit_indices.append(i)\n digit_values.append(int(s))\n\n n = len(digit_values)\n if n \u003c _PAGE_SEQUENCE_MIN_LENGTH:\n return set()\n\n tails = []\n tails_idx = []\n parents = [-1] * n\n\n for i, v in enumerate(digit_values):\n pos = bisect.bisect_right(tails, v)\n if pos > 0:\n parents[i] = tails_idx[pos - 1]\n if pos == len(tails):\n tails.append(v)\n tails_idx.append(i)\n else:\n tails[pos] = v\n tails_idx[pos] = i\n\n lnds = []\n cur = tails_idx[-1]\n while cur != -1:\n lnds.append(cur)\n cur = parents[cur]\n lnds.reverse()\n\n if len(lnds) \u003c _PAGE_SEQUENCE_MIN_LENGTH:\n return set()\n if len(lnds) / n \u003c _PAGE_SEQUENCE_MIN_RATIO:\n return set()\n\n return {digit_indices[i] for i in lnds}\n\n\n_CODE_FENCE_RE = re.compile(r'^(`{3,}|~{3,})')\n\n\ndef _fenced_code_line_indices(lines):\n found = set()\n fence = None\n for i, line in enumerate(lines):\n match = _CODE_FENCE_RE.match(line)\n if fence:\n found.add(i)\n if (match and match.group(1)[0] == fence[0] and len(match.group(1)) >= len(fence)\n and not line[match.end():].strip()):\n fence = None\n elif match:\n fence = match.group(1)\n found.add(i)\n return found\n\n\ndef _is_empty_anchor_line(line):\n \"\"\"True for a body line that only holds empty anchors, e.g. `[]` or `[]{#page_3}`.\n\n Indented lines are code, never anchors; fenced code is excluded by the caller.\n \"\"\"\n if line[:1].isspace():\n return False\n residue = re.sub(r'\\{[^}]*\\}|\$#[^)\\s]*\$', '', line)\n return '[' in residue and not residue.strip('[] \\t')\n\n\ndef clean_calibre_markers(content, strip_page_numbers=False, detect_page_numbers=True):\n \"\"\"Clean up Calibre-specific markers from markdown content.\n\n With detect_page_numbers, standalone digit lines are handled in two layers:\n 1. If a line is adjacent to Calibre noise (::: fence, .ct}/.cn} marker),\n drop it — clearly leftover.\n 2. Otherwise, run LNDS over all standalone digits to detect a monotonic\n page-number sequence and drop those. Outliers like years (1984),\n chapter numbers, and citation indices stay preserved.\n\n detect_page_numbers=False keeps every standalone digit line: it is meant\n for formats whose page numbers are not text (EPUB anchors, DOCX footers),\n where such lines are chapter, verse or exercise numbers.\n\n Pass strip_page_numbers=True to bypass both layers and aggressively delete\n every standalone-digit line (legacy behavior).\n\n Lines holding only empty anchors are dropped as well, but they still count\n as neighbours for layer 1.\n \"\"\"\n content = re.sub(r'\\{\\.calibre[^}]*\\}', '', content)\n content = re.sub(r'\$#calibre_link-\\d+\$', '', content)\n\n # Clean heading calibre attribute blocks: {#calibre_link-N .calibreN}\n content = re.sub(r'\\s*\\{#calibre_link-\\d+[^}]*\\}', '', content)\n\n # Clean [**text**] format to **text**\n content = re.sub(r'\\[\\*\\*([^*]+)\\*\\*\\]', r'**\\1**', content)\n\n lines = content.split('\\n')\n\n page_number_lines = set() if strip_page_numbers else _detect_page_number_lines(lines)\n code_lines = _fenced_code_line_indices(lines)\n\n def is_calibre_noise(line):\n s = line.strip()\n if not s:\n return False\n if s.startswith(':::'):\n return True\n if s.endswith('.ct}') or s.endswith('.cn}'):\n return True\n return False\n\n def prev_nonblank(idx):\n for j in range(idx - 1, -1, -1):\n if lines[j].strip():\n return lines[j]\n return None\n\n def next_nonblank(idx):\n for j in range(idx + 1, len(lines)):\n if lines[j].strip():\n return lines[j]\n return None\n\n cleaned_lines = []\n for i, line in enumerate(lines):\n stripped_line = line.strip()\n\n if stripped_line.startswith(':::'):\n continue\n if stripped_line.endswith('.ct}') or stripped_line.endswith('.cn}'):\n continue\n if i not in code_lines and _is_empty_anchor_line(line):\n continue\n\n if re.match(r'^\\s*\\d+\\s*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, line):\n if strip_page_numbers:\n continue\n if detect_page_numbers:\n if i in page_number_lines:\n continue\n prev = prev_nonblank(i)\n nxt = next_nonblank(i)\n if (prev is not None and is_calibre_noise(prev)) or \\\n (nxt is not None and is_calibre_noise(nxt)):\n continue\n # else: preserve as real content\n\n cleaned_lines.append(line)\n\n content = '\\n'.join(cleaned_lines)\n content = re.sub(r'\\n{3,}', '\\n\\n', content)\n return content\n\n\n# =============================================================================\n# Markdown input: pre-extracted Markdown (e.g. from MinerU / Marker) skips Calibre\n# =============================================================================\n\nMARKDOWN_EXTENSIONS = ('.md', '.markdown')\nCALIBRE_CONVERSION_METHOD = \"calibre_htmlz\"\nMARKDOWN_CONVERSION_METHOD = \"markdown\"\n\n_MD_IMG_RE = re.compile(r'(!\\[[^\\]]*\\]\\(\\s*)(\u003c[^>]+>|[^)\\s]+)')\n_HTML_IMG_RE = re.compile(r'(\u003cimg\\b[^>]*?\\bsrc\\s*=\\s*)([\"\\'])(.*?)\\2', re.IGNORECASE | re.DOTALL)\n_DATA_URI_RE = re.compile(r'^data:image/([A-Za-z0-9.+-]+);base64,(.*)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, re.DOTALL)\n_URI_SCHEME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9+.-]*:')\n_DATA_URI_EXTENSIONS = {\n \"jpeg\": \".jpg\", \"jpg\": \".jpg\", \"png\": \".png\", \"gif\": \".gif\",\n \"webp\": \".webp\", \"svg+xml\": \".svg\", \"bmp\": \".bmp\",\n}\n\n_FRONT_MATTER_RE = re.compile(r'\\A---[ \\t]*\\n(.*?)\\n(?:---|\\.\\.\\.)[ \\t]*(?:\\n|\\Z)', re.DOTALL)\n_FRONT_MATTER_KEYS = {\n \"title\": \"title\",\n \"author\": \"creator\",\n \"creator\": \"creator\",\n \"publisher\": \"publisher\",\n \"lang\": \"language\",\n \"language\": \"language\",\n}\n\n\ndef _unquote_yaml_scalar(value):\n value = value.strip()\n if len(value) >= 2 and value[0] == value[-1] and value[0] in ('\"', \"'\"):\n return value[1:-1]\n return value\n\n\ndef split_front_matter(content):\n \"\"\"Split a leading YAML front matter block off content.\n\n Returns (metadata, body). Only flat `key: value` scalars and `- item`\n lists for the keys in _FRONT_MATTER_KEYS are read; anything fancier is\n ignored. A leading `---` block without any `key:` line is left alone,\n since it is more likely a thematic break than metadata.\n \"\"\"\n match = _FRONT_MATTER_RE.match(content)\n if not match:\n return {}, content\n lines = match.group(1).split('\\n')\n if not any(re.match(r'^[A-Za-z_][\\w-]*\\s*:', line) for line in lines):\n return {}, content\n\n metadata = {}\n current_key = None\n for line in lines:\n key_match = re.match(r'^([A-Za-z_][\\w-]*)\\s*:\\s*(.*)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, line)\n if key_match:\n key = _FRONT_MATTER_KEYS.get(key_match.group(1).lower())\n value = _unquote_yaml_scalar(key_match.group(2))\n current_key = None\n if key and key not in metadata:\n if value:\n metadata[key] = value\n else:\n current_key = key # value follows as a `- item` list\n continue\n item_match = re.match(r'^\\s+-\\s+(.+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, line)\n if current_key and item_match:\n item = _unquote_yaml_scalar(item_match.group(1))\n if item:\n existing = metadata.get(current_key)\n metadata[current_key] = f\"{existing}, {item}\" if existing else item\n return metadata, content[match.end():]\n\n\ndef _first_h1_title(content):\n \"\"\"Return the text of the document's first heading if it is level 1.\n\n Layout parsers put a paper's title first, as `# Title`. If the first\n heading is deeper than H1 it is a section, not the title, so return None.\n \"\"\"\n in_fence = False\n for line in content.split('\\n'):\n stripped = line.strip()\n if stripped.startswith('```'):\n in_fence = not in_fence\n continue\n if in_fence:\n continue\n heading = re.match(r'^(#{1,6})\\s+(.*?)\\s*#*\\s*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, stripped)\n if heading:\n if len(heading.group(1)) != 1:\n return None\n title = re.sub(r'\\s*\\{[^}]*\\}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, '', heading.group(2)).strip()\n return title or None\n return None\n\n\ndef markdown_metadata(content):\n \"\"\"Book metadata for Markdown input: front matter first, then the first H1.\"\"\"\n metadata, body = split_front_matter(content.replace('', ''))\n if 'title' not in metadata:\n title = _first_h1_title(body)\n if title:\n metadata['title'] = title\n return metadata\n\n\ndef import_markdown(md_path, temp_dir):\n \"\"\"Return Markdown content ready for chunking, with images moved under temp_dir/images/.\n\n Relative image paths are resolved against the Markdown file's directory\n and copied in; base64 data URIs (MinerU inlines images that way) are\n decoded to files so chunks handed to translators never carry image\n payloads. Remote URLs stay untouched. YAML front matter is dropped from\n the body — its metadata goes to config.txt via markdown_metadata().\n \"\"\"\n with open(md_path, 'r', encoding='utf-8') as f:\n content = f.read()\n content = content.replace('', '').replace(' ', ' ')\n _, content = split_front_matter(content)\n\n md_dir = os.path.dirname(os.path.abspath(md_path))\n images_dir = os.path.join(temp_dir, \"images\")\n refs = {}\n stored = {}\n embedded_count = 0\n\n def store(name, data):\n base, ext = os.path.splitext(name)\n digest = hashlib.sha256(data).hexdigest()\n candidate, n = name, 1\n while candidate in stored and stored[candidate] != digest:\n n += 1\n candidate = f\"{base}-{n}{ext}\"\n if candidate not in stored:\n os.makedirs(images_dir, exist_ok=True)\n with open(os.path.join(images_dir, candidate), 'wb') as out:\n out.write(data)\n stored[candidate] = digest\n return f\"images/{candidate}\"\n\n def localize(target):\n nonlocal embedded_count\n raw = target[1:-1] if target.startswith('\u003c') and target.endswith('>') else target\n if raw in refs:\n return refs[raw]\n new_ref = target\n data_uri = _DATA_URI_RE.match(raw)\n if data_uri:\n try:\n data = base64.b64decode(data_uri.group(2))\n except (binascii.Error, ValueError):\n data = b''\n if data:\n subtype = data_uri.group(1).lower()\n ext = _DATA_URI_EXTENSIONS.get(subtype, '.' + re.sub(r'[^a-z0-9]', '', subtype))\n embedded_count += 1\n new_ref = store(f\"embedded-{embedded_count:04d}{ext}\", data)\n else:\n print(\"Warning: could not decode an embedded data-URI image; left inline\")\n elif not _URI_SCHEME_RE.match(raw) and not raw.startswith('#'):\n src = os.path.normpath(os.path.join(md_dir, urllib.parse.unquote(raw)))\n if os.path.isfile(src):\n with open(src, 'rb') as f:\n new_ref = store(os.path.basename(src), f.read())\n else:\n print(f\"Warning: image not found next to the Markdown file, reference kept as-is: {raw}\")\n refs[raw] = new_ref\n return new_ref\n\n content = _MD_IMG_RE.sub(lambda m: m.group(1) + localize(m.group(2)), content)\n content = _HTML_IMG_RE.sub(\n lambda m: f\"{m.group(1)}{m.group(2)}{localize(m.group(3))}{m.group(2)}\", content\n )\n\n if stored:\n print(f\"Localized {len(stored)} image(s) into {images_dir}/\")\n return content\n\n\ndef import_markdown_input(input_file, input_md, temp_dir):\n \"\"\"Copy a Markdown source into temp_dir/input.md with its images localized.\"\"\"\n os.makedirs(temp_dir, exist_ok=True)\n content = import_markdown(input_file, temp_dir)\n if not content.strip():\n print(f\"Error: {input_file} has no content to translate\")\n return False\n with open(input_md, 'w', encoding='utf-8') as f:\n f.write(content)\n print(f\"Markdown import successful: {input_md}\")\n return True\n\n\n# =============================================================================\n# Structural block parsing and chunk splitting (Step 3)\n# =============================================================================\n\n# Pandoc footnote labels cannot contain whitespace, ^, [ or ].\n_FOOTNOTE_DEF_RE = re.compile(r'^ {0,3}\\[\\^[^\\]\\s^\\[]+\\]:')\n\n\ndef _is_indented_text(line):\n return line.startswith((' ', '\\t')) and bool(line.strip())\n\n\ndef parse_structural_blocks(content):\n \"\"\"Parse markdown into structural blocks that should not be split.\n\n Returns list of (text, block_type) tuples where block_type is one of:\n 'heading', 'code_block', 'math_block', 'table', 'list', 'blockquote',\n 'footnote', 'image', 'paragraph'\n \"\"\"\n blocks = []\n lines = content.split('\\n')\n i = 0\n\n while i \u003c len(lines):\n line = lines[i]\n stripped = line.strip()\n\n # Code block (fenced)\n if stripped.startswith('```'):\n block_lines = [line]\n i += 1\n while i \u003c len(lines):\n block_lines.append(lines[i])\n if lines[i].strip().startswith('```') and len(block_lines) > 1:\n i += 1\n break\n i += 1\n blocks.append(('\\n'.join(block_lines), 'code_block'))\n continue\n\n # Display math ($ ... $). Lines inside may look like list items or\n # tables, so the block must be kept whole. Like pandoc, never pair\n # delimiters across a blank line — an unclosed $ must not swallow\n # the rest of the chapter.\n if stripped.startswith('$'):\n if len(stripped) >= 4 and stripped.endswith('$'):\n blocks.append((line, 'math_block'))\n i += 1\n continue\n end = None\n for j in range(i + 1, len(lines)):\n s = lines[j].strip()\n if not s:\n break\n if s.endswith('$'):\n end = j\n break\n if end is not None:\n blocks.append(('\\n'.join(lines[i:end + 1]), 'math_block'))\n i = end + 1\n continue\n\n # Heading\n if re.match(r'^#{1,6}\\s', stripped):\n blocks.append((line, 'heading'))\n i += 1\n continue\n\n # Blockquote\n if stripped.startswith('>'):\n block_lines = [line]\n i += 1\n while i \u003c len(lines) and (lines[i].strip().startswith('>') or\n (lines[i].strip() and not re.match(r'^#{1,6}\\s', lines[i].strip())\n and not lines[i].strip().startswith('```')\n and not lines[i].strip().startswith('|')\n and not re.match(r'^[-*+]\\s', lines[i].strip())\n and not re.match(r'^\\d+\\.\\s', lines[i].strip())\n and block_lines[-1].strip().startswith('>'))):\n block_lines.append(lines[i])\n i += 1\n blocks.append(('\\n'.join(block_lines), 'blockquote'))\n continue\n\n # Table (lines starting with |)\n if stripped.startswith('|'):\n block_lines = [line]\n i += 1\n while i \u003c len(lines) and lines[i].strip().startswith('|'):\n block_lines.append(lines[i])\n i += 1\n blocks.append(('\\n'.join(block_lines), 'table'))\n continue\n\n # List (unordered or ordered)\n if re.match(r'^[-*+]\\s', stripped) or re.match(r'^\\d+\\.\\s', stripped):\n block_lines = [line]\n i += 1\n while i \u003c len(lines):\n s = lines[i].strip()\n # Continue list: list items, indented continuation, or blank lines within list\n if (re.match(r'^[-*+]\\s', s) or re.match(r'^\\d+\\.\\s', s) or\n (lines[i].startswith(' ') and s) or\n (s == '' and i + 1 \u003c len(lines) and\n (re.match(r'^[-*+]\\s', lines[i+1].strip()) or\n re.match(r'^\\d+\\.\\s', lines[i+1].strip()) or\n lines[i+1].startswith(' ')))):\n block_lines.append(lines[i])\n i += 1\n else:\n break\n blocks.append(('\\n'.join(block_lines), 'list'))\n continue\n\n # Footnote definition plus its indented continuation paragraphs. Kept\n # whole: merging strips each chunk, which would un-indent a\n # continuation that started the next chunk and detach it from its note.\n if _FOOTNOTE_DEF_RE.match(line):\n block_lines = [line]\n i += 1\n while i \u003c len(lines):\n if _is_indented_text(lines[i]) or (\n not lines[i].strip() and i + 1 \u003c len(lines) and _is_indented_text(lines[i + 1])\n ):\n block_lines.append(lines[i])\n i += 1\n else:\n break\n blocks.append(('\\n'.join(block_lines), 'footnote'))\n continue\n\n # Image line (standalone or with surrounding caption)\n if re.match(r'!\\[', stripped):\n blocks.append((line, 'image'))\n i += 1\n continue\n\n # Empty line — just a paragraph separator\n if stripped == '':\n blocks.append((line, 'paragraph'))\n i += 1\n continue\n\n # Regular paragraph — collect contiguous non-empty, non-special lines\n block_lines = [line]\n i += 1\n while i \u003c len(lines):\n s = lines[i].strip()\n if (s == '' or s.startswith('```') or s.startswith('$') or re.match(r'^#{1,6}\\s', s) or\n s.startswith('>') or s.startswith('|') or\n re.match(r'^[-*+]\\s', s) or re.match(r'^\\d+\\.\\s', s) or\n re.match(r'!\\[', s)):\n break\n block_lines.append(lines[i])\n i += 1\n blocks.append(('\\n'.join(block_lines), 'paragraph'))\n continue\n\n return blocks\n\n\ndef merge_blocks_to_chunks(blocks, target_size=6000):\n \"\"\"Merge structural blocks into chunks respecting target_size.\n\n Prefers to split at heading boundaries. Never splits within a single\n structural block unless the block itself exceeds target_size * 2.\n \"\"\"\n chunks = []\n current_parts = []\n current_size = 0\n\n def flush():\n nonlocal current_parts, current_size\n if current_parts:\n chunks.append('\\n'.join(current_parts))\n current_parts = []\n current_size = 0\n\n for text, btype in blocks:\n block_size = len(text)\n\n # If a single block is oversized, handle degradation\n if block_size > target_size * 2:\n flush()\n if btype == 'footnote':\n # A continuation paragraph cut loose from its `[^N]:` line\n # loses its indent when chunks are merged and becomes body text.\n print(f\" WARNING: Oversized footnote ({block_size} chars), kept in one chunk\")\n chunks.append(text)\n continue\n print(f\" WARNING: Oversized {btype} block ({block_size} chars), force-splitting\")\n sub_chunks = _force_split_block(text, target_size)\n chunks.extend(sub_chunks)\n continue\n\n # Prefer to split at heading boundaries\n if btype == 'heading' and current_size > 0:\n flush()\n\n # Would adding this block exceed target?\n if current_size + block_size > target_size and current_parts:\n flush()\n\n current_parts.append(text)\n current_size += block_size\n\n flush()\n # The blank line after a final oversized block would otherwise be its own\n # chunk, and an empty translation of it fails the merge.\n return [chunk for chunk in chunks if chunk.strip()]\n\n\ndef _force_split_block(text, target_size):\n \"\"\"Force-split an oversized block by paragraph (empty lines), then by lines.\n\n For fenced code blocks, each resulting chunk gets proper opening/closing fences\n so it remains valid Markdown.\n \"\"\"\n stripped = text.strip()\n is_fenced_code = stripped.startswith('```')\n\n # Extract fence info for code blocks\n fence_opener = ''\n if is_fenced_code:\n first_line = stripped.split('\\n', 1)[0]\n fence_opener = first_line # e.g. \"```python\"\n\n # Try splitting by empty lines first (not applicable for code blocks — no empty lines expected)\n if not is_fenced_code:\n paragraphs = re.split(r'\\n\\n+', text)\n if len(paragraphs) > 1:\n chunks = []\n current = []\n current_size = 0\n for para in paragraphs:\n para_size = len(para)\n if current_size + para_size > target_size and current:\n chunks.append('\\n\\n'.join(current))\n current = [para]\n current_size = para_size\n else:\n current.append(para)\n current_size += para_size\n if current:\n chunks.append('\\n\\n'.join(current))\n return chunks\n\n # Split by lines\n lines = text.split('\\n')\n\n # For code blocks, strip the opening and closing fences before splitting content\n if is_fenced_code:\n # Remove opening fence line\n content_lines = lines[1:]\n # Remove closing fence line if present\n if content_lines and content_lines[-1].strip().startswith('```'):\n content_lines = content_lines[:-1]\n lines = content_lines\n\n chunks = []\n current = []\n current_size = 0\n for line in lines:\n line_size = len(line) + 1\n if current_size + line_size > target_size and current:\n chunks.append('\\n'.join(current))\n current = [line]\n current_size = line_size\n else:\n current.append(line)\n current_size += line_size\n if current:\n chunks.append('\\n'.join(current))\n\n # Re-wrap each chunk in fences for code blocks\n if is_fenced_code:\n chunks = [f\"{fence_opener}\\n{chunk}\\n```\" for chunk in chunks]\n\n return chunks\n\n\ndef split_markdown_structured(md_file, temp_dir, target_size=6000):\n \"\"\"Split markdown into structural chunks.\n\n Returns list of chunk filenames (e.g. ['chunk0001.md', ...]).\n \"\"\"\n try:\n with open(md_file, 'r', encoding='utf-8') as f:\n content = f.read()\n\n blocks = parse_structural_blocks(content)\n chunk_texts = merge_blocks_to_chunks(blocks, target_size)\n\n chunk_files = []\n for i, chunk_text in enumerate(chunk_texts, 1):\n filename = f\"chunk{i:04d}.md\"\n chunk_file = os.path.join(temp_dir, filename)\n with open(chunk_file, 'w', encoding='utf-8') as f:\n f.write(chunk_text)\n chunk_files.append(filename)\n\n print(f\"Split into {len(chunk_files)} chunks\")\n for filename in chunk_files:\n filepath = os.path.join(temp_dir, filename)\n size = os.path.getsize(filepath)\n print(f\" {filename}: {size} characters\")\n\n return chunk_files\n except Exception as e:\n print(f\"Error splitting markdown: {e}\")\n return []\n\n\ndef _find_existing_chunk_files(temp_dir):\n \"\"\"Find existing chunk source filenames (excluding output_ prefixed), sorted.\"\"\"\n chunk_files = glob.glob(os.path.join(temp_dir, 'chunk*.md'))\n chunk_files = [os.path.basename(f) for f in chunk_files if not os.path.basename(f).startswith('output_')]\n return sorted(chunk_files)\n\n\ndef create_config_file(temp_dir, input_file, input_lang, output_lang, metadata=None,\n conversion_method=CALIBRE_CONVERSION_METHOD):\n \"\"\"Create config.txt file for the pipeline\"\"\"\n try:\n config_file = os.path.join(temp_dir, \"config.txt\")\n\n config_content = f\"\"\"# Translation Configuration\ninput_file={input_file}\ninput_lang={input_lang}\noutput_lang={output_lang}\nconversion_method={conversion_method}\n\"\"\"\n if metadata:\n config_content += f\"\\n# Book Metadata\\n\"\n if 'title' in metadata:\n config_content += f\"original_title={metadata['title']}\\n\"\n if 'creator' in metadata:\n config_content += f\"creator={metadata['creator']}\\n\"\n if 'publisher' in metadata:\n config_content += f\"publisher={metadata['publisher']}\\n\"\n if 'language' in metadata:\n config_content += f\"source_language={metadata['language']}\\n\"\n\n with open(config_file, 'w', encoding='utf-8') as f:\n f.write(config_content)\n\n print(f\"Created config file: {config_file}\")\n return True\n except Exception as e:\n print(f\"Error creating config file: {e}\")\n return False\n\n\ndef _do_split_and_manifest(temp_dir, input_md, chunk_size):\n \"\"\"Split markdown and create manifest. Returns chunk count or 0 on failure.\"\"\"\n existing = _find_existing_chunk_files(temp_dir)\n if existing:\n print(f\"Skipping markdown splitting - found {len(existing)} existing chunk files\")\n # Create/update manifest for existing files\n create_manifest(temp_dir, existing, input_md)\n return len(existing)\n\n chunk_files = split_markdown_structured(input_md, temp_dir, chunk_size)\n if not chunk_files:\n return 0\n create_manifest(temp_dir, chunk_files, input_md)\n return len(chunk_files)\n\n\ndef _check_strip_page_numbers_cache_conflict(strip_flag, temp_dir, input_md):\n \"\"\"Return cached files that would silently neutralize a page-number flag.\n\n --strip-page-numbers and --detect-page-numbers only take effect inside\n clean_calibre_markers, which runs during HTML→Markdown conversion. If\n input.md or chunk*.md already exist from a prior run, both are reused\n as-is and the flag becomes a no-op. Surface that conflict so the user\n knows to clean up.\n \"\"\"\n if not strip_flag:\n return []\n if not os.path.isdir(temp_dir):\n return []\n\n blockers = []\n if os.path.exists(input_md):\n blockers.append(input_md)\n\n existing_chunks = [\n f for f in glob.glob(os.path.join(temp_dir, 'chunk*.md'))\n if not os.path.basename(f).startswith('output_')\n ]\n if existing_chunks:\n blockers.append(f\"{len(existing_chunks)} chunk file(s) under {temp_dir}/\")\n\n return blockers\n\n\ndef _abort_on_strip_cache_conflict(blockers, temp_dir):\n if not blockers:\n return\n print(\"Error: page-number options cannot take effect because cached files exist:\")\n for b in blockers:\n print(f\" - {b}\")\n print(f\"Delete the cached files (or remove the entire {temp_dir}/ directory) and re-run.\")\n sys.exit(1)\n\n\ndef main():\n \"\"\"Main conversion function\"\"\"\n parser = argparse.ArgumentParser(\n description=\"Convert PDF/DOCX/EPUB (via Calibre HTMLZ) or Markdown to markdown chunks\"\n )\n parser.add_argument(\n \"input_file\",\n help=\"Input file: PDF, DOCX, or EPUB (converted by Calibre), or Markdown (.md/.markdown), \"\n \"e.g. the output of a layout-aware PDF parser such as MinerU or Marker\",\n )\n parser.add_argument(\"-l\", \"--ilang\", default=\"auto\", help=\"Input language (default: auto)\")\n parser.add_argument(\"--olang\", default=\"zh\", help=\"Output language (default: zh)\")\n parser.add_argument(\"--chunk-size\", type=int, default=6000, help=\"Target chunk size in characters (default: 6000)\")\n parser.add_argument(\n \"--temp-root\",\n default=None,\n help=\"Directory under which {book_name}_temp/ will be created (default: current working directory)\",\n )\n parser.add_argument(\n \"--strip-page-numbers\",\n action=\"store_true\",\n help=\"Aggressively delete every standalone-digit line (legacy behavior).\",\n )\n parser.add_argument(\n \"--detect-page-numbers\",\n action=\"store_true\",\n help=\"Drop detected page-number lines from EPUB/DOCX too. By default only \"\n \"PDF input is cleaned this way; EPUB/DOCX keep every standalone number.\",\n )\n\n args = parser.parse_args()\n input_file = args.input_file\n\n if not os.path.exists(input_file):\n print(f\"Error: Input file not found: {input_file}\")\n sys.exit(1)\n\n file_ext = os.path.splitext(input_file)[1].lower()\n is_markdown = file_ext in MARKDOWN_EXTENSIONS\n if file_ext not in ['.pdf', '.docx', '.epub'] and not is_markdown:\n print(f\"Error: Unsupported file type: {file_ext}\")\n sys.exit(1)\n page_number_flag = args.strip_page_numbers or args.detect_page_numbers\n if is_markdown and page_number_flag:\n flag = \"--strip-page-numbers\" if args.strip_page_numbers else \"--detect-page-numbers\"\n print(f\"Error: {flag} only cleans Calibre output; it does not apply to Markdown input.\")\n sys.exit(1)\n # Only PDFs carry page numbers as text. EPUB page numbers are invisible\n # anchors and Calibre drops DOCX footers, so their digit lines are content.\n detect_page_numbers = args.detect_page_numbers or file_ext == '.pdf'\n\n if is_markdown:\n print(\"=== Markdown Import (no Calibre) ===\")\n else:\n print(\"=== File Conversion via Calibre HTMLZ ===\")\n print(f\"Input file: {input_file}\")\n print(f\"Target chunk size: {args.chunk_size} characters\")\n if args.temp_root:\n print(f\"Temp root: {args.temp_root}\")\n\n try:\n temp_dir = build_temp_dir(input_file, args.temp_root)\n current_fingerprint = source_fingerprint(input_file)\n _abort_on_source_cache_mismatch(\n *check_source_cache(temp_dir, current_fingerprint), temp_dir=temp_dir\n )\n\n if is_markdown:\n input_md = os.path.join(temp_dir, \"input.md\")\n with open(input_file, 'r', encoding='utf-8') as f:\n metadata = markdown_metadata(f.read())\n if os.path.exists(input_md):\n print(\"Skipping Markdown import - input.md already exists\")\n elif not import_markdown_input(input_file, input_md, temp_dir):\n sys.exit(1)\n\n chunk_count = _do_split_and_manifest(temp_dir, input_md, args.chunk_size)\n if chunk_count == 0:\n sys.exit(1)\n\n create_config_file(temp_dir, input_file, args.ilang, args.olang, metadata,\n conversion_method=MARKDOWN_CONVERSION_METHOD)\n _write_source_fingerprint(temp_dir, current_fingerprint)\n print(\"Conversion completed successfully!\")\n print(f\"Temp directory: {temp_dir}\")\n print(f\"Markdown chunks: {chunk_count} files\")\n return\n\n calibre_path = find_calibre_convert()\n if not calibre_path:\n print(\"Error: Calibre ebook-convert not found\")\n print(\"Please install Calibre: https://calibre-ebook.com/\")\n sys.exit(1)\n\n input_html_path = os.path.join(temp_dir, \"input.html\")\n\n if os.path.exists(input_html_path):\n print(f\"Skipping HTMLZ conversion - input.html already exists\")\n\n metadata = {}\n config_file = os.path.join(temp_dir, \"config.txt\")\n if os.path.exists(config_file):\n try:\n with open(config_file, 'r', encoding='utf-8') as f:\n for line in f:\n if '=' in line:\n key, value = line.strip().split('=', 1)\n if key == 'original_title':\n metadata['title'] = value\n elif key == 'creator':\n metadata['creator'] = value\n elif key == 'publisher':\n metadata['publisher'] = value\n elif key == 'source_language':\n metadata['language'] = value\n except Exception as e:\n print(f\"Warning: Could not read metadata from config: {e}\")\n\n input_md = os.path.join(temp_dir, \"input.md\")\n _abort_on_strip_cache_conflict(\n _check_strip_page_numbers_cache_conflict(page_number_flag, temp_dir, input_md),\n temp_dir,\n )\n if os.path.exists(input_md):\n print(f\"Skipping HTML to Markdown conversion - input.md already exists\")\n else:\n if not convert_html_to_markdown(input_html_path, input_md,\n strip_page_numbers=args.strip_page_numbers,\n detect_page_numbers=detect_page_numbers):\n sys.exit(1)\n\n chunk_count = _do_split_and_manifest(temp_dir, input_md, args.chunk_size)\n if chunk_count == 0:\n sys.exit(1)\n\n create_config_file(temp_dir, input_file, args.ilang, args.olang, metadata)\n _write_source_fingerprint(temp_dir, current_fingerprint)\n print(\"Conversion completed successfully!\")\n print(f\"Temp directory: {temp_dir}\")\n return\n\n with tempfile.TemporaryDirectory() as extract_dir:\n # Never next to the source: that path may be the user's own\n # \u003cbook>.htmlz, which Calibre would overwrite and we would delete.\n htmlz_name = os.path.splitext(os.path.basename(input_file))[0] + \".htmlz\"\n htmlz_file = os.path.join(extract_dir, htmlz_name)\n if not convert_to_htmlz(input_file, htmlz_file, calibre_path):\n sys.exit(1)\n\n html_file, images_dir = extract_htmlz(htmlz_file, extract_dir)\n if not html_file:\n sys.exit(1)\n\n metadata = extract_metadata_from_htmlz(extract_dir)\n\n temp_dir = setup_temp_directory(input_file, html_file, images_dir, temp_root=args.temp_root)\n if not temp_dir:\n sys.exit(1)\n\n input_html = os.path.join(temp_dir, \"input.html\")\n input_md = os.path.join(temp_dir, \"input.md\")\n\n _abort_on_strip_cache_conflict(\n _check_strip_page_numbers_cache_conflict(page_number_flag, temp_dir, input_md),\n temp_dir,\n )\n if os.path.exists(input_md):\n print(f\"Skipping HTML to Markdown conversion - input.md already exists\")\n else:\n if not convert_html_to_markdown(input_html, input_md,\n strip_page_numbers=args.strip_page_numbers,\n detect_page_numbers=detect_page_numbers):\n sys.exit(1)\n\n chunk_count = _do_split_and_manifest(temp_dir, input_md, args.chunk_size)\n if chunk_count == 0:\n sys.exit(1)\n\n create_config_file(temp_dir, input_file, args.ilang, args.olang, metadata)\n _write_source_fingerprint(temp_dir, current_fingerprint)\n\n print(\"Conversion completed successfully!\")\n print(f\"Temp directory: {temp_dir}\")\n print(f\"Markdown chunks: {chunk_count} files\")\n\n except KeyboardInterrupt:\n print(\"\\nConversion interrupted by user\")\n sys.exit(1)\n except Exception as e:\n print(f\"Error: {e}\")\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":60151,"content_sha256":"201b4dadcfcfc3beddb908d529da3c67a77991f79fc3198147f95411cc5ab162"},{"filename":"scripts/glossary.py","content":"#!/usr/bin/env python3\n\"\"\"\nglossary.py - Term-consistency glossary for parallel-subagent translation.\n\nA separate sub-agent translates each chunk with a fresh context. Without\nshared state, the same proper noun can get translated three different ways\nacross a 100-chunk book. This module manages a hand-editable glossary.json\nthat the main agent injects into each sub-agent's prompt as a per-chunk term\ntable — so every sub-agent sees the same canonical translations for the\nterms that matter to its chunk.\n\nThe schema (v2):\n\n {\n \"version\": 2,\n \"terms\": [\n {\"id\": \"Manhattan\", \"source\": \"Manhattan\", \"target\": \"曼哈顿\",\n \"category\": \"place\", \"aliases\": [], \"gender\": \"unknown\",\n \"confidence\": \"medium\", \"frequency\": 12,\n \"evidence_refs\": [], \"notes\": \"\"}\n ],\n \"high_frequency_top_n\": 20,\n \"applied_meta_hashes\": {}\n }\n\nLives at \u003ctemp_dir>/glossary.json and is meant to be hand-edited.\n\"\"\"\n\nimport argparse\nimport hashlib\nimport json\nimport os\nimport re\nimport sys\nimport tempfile\nfrom pathlib import Path\n\n\nGLOSSARY_SCHEMA_VERSION = 2\nDEFAULT_TOP_N = 20\nDEFAULT_MAX_TERMS = 50\n\nVALID_GENDERS = ('male', 'female', 'nonbinary', 'unknown')\nVALID_CONFIDENCES = ('low', 'medium', 'high')\n\n_HEX_RE = re.compile(r'^[0-9a-f]+

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

)\n\n_CJK_RANGES = (\n (0x3400, 0x4DBF), # CJK Extension A\n (0x4E00, 0x9FFF), # CJK Unified Ideographs\n (0x3040, 0x309F), # Hiragana\n (0x30A0, 0x30FF), # Katakana\n (0xAC00, 0xD7AF), # Hangul Syllables\n)\n\n\ndef _contains_cjk(s):\n for c in s:\n cp = ord(c)\n for lo, hi in _CJK_RANGES:\n if lo \u003c= cp \u003c= hi:\n return True\n return False\n\n\ndef _canonical_json(data):\n \"\"\"Stable JSON for hashing — sorted keys, no whitespace, unicode preserved.\"\"\"\n return json.dumps(data, sort_keys=True, separators=(',', ':'), ensure_ascii=False)\n\n\ndef glossary_hash(glossary):\n \"\"\"SHA-256 of the canonical glossary. Input order / whitespace insensitive.\"\"\"\n return hashlib.sha256(_canonical_json(glossary).encode('utf-8')).hexdigest()\n\n\ndef term_hash(term):\n \"\"\"SHA-256 of a single term's prompt-affecting fields.\n\n Aliases participate because they are injected into the per-chunk term\n table (别名 column): adding or editing an alias changes the prompt\n contract for chunks that already selected this term, so run_state must\n see a different hash and plan a re-translation. The alias segment is\n appended only when aliases exist, so alias-less terms keep their\n historical hash and existing run_state.json records stay valid.\n \"\"\"\n payload = f\"{term.get('source', '')}→{term.get('target', '')}|{term.get('category', '')}\"\n aliases = sorted(term.get('aliases', []) or [])\n if aliases:\n payload += '|aliases=' + json.dumps(aliases, ensure_ascii=False)\n return hashlib.sha256(payload.encode('utf-8')).hexdigest()\n\n\ndef _v2_term_defaults(term):\n \"\"\"Apply v2 defaults to a term in place. Caller must have already\n validated v1-shape required fields.\"\"\"\n term.setdefault('id', term.get('source', ''))\n term.setdefault('aliases', [])\n term.setdefault('gender', 'unknown')\n term.setdefault('confidence', 'medium')\n term.setdefault('evidence_refs', [])\n term.setdefault('notes', '')\n term.setdefault('frequency', 0)\n\n\ndef _validate_term_strict(term, idx, path):\n \"\"\"Strict v2 validation for a single term. Raises ValueError mentioning\n the file `path` for context.\"\"\"\n if not isinstance(term, dict):\n raise ValueError(f\"Glossary term #{idx} in {path} must be an object\")\n\n for required in ('source', 'target'):\n if required not in term:\n raise ValueError(\n f\"Glossary term #{idx} in {path} missing required field '{required}'\"\n )\n if not isinstance(term[required], str):\n raise ValueError(\n f\"Glossary term #{idx} in {path}: field '{required}' must be a string, \"\n f\"got {type(term[required]).__name__}\"\n )\n\n if 'category' in term and not isinstance(term['category'], str):\n raise ValueError(\n f\"Glossary term #{idx} in {path}: field 'category' must be a string, \"\n f\"got {type(term['category']).__name__}\"\n )\n\n if 'frequency' in term:\n if isinstance(term['frequency'], bool) or not isinstance(term['frequency'], int):\n raise ValueError(\n f\"Glossary term #{idx} in {path}: field 'frequency' must be an integer, \"\n f\"got {type(term['frequency']).__name__}\"\n )\n\n if 'id' not in term:\n raise ValueError(f\"Glossary term #{idx} in {path} missing required field 'id'\")\n if not isinstance(term['id'], str) or not term['id']:\n raise ValueError(\n f\"Glossary term #{idx} in {path}: 'id' must be a non-empty string\"\n )\n\n aliases = term.get('aliases', [])\n if not isinstance(aliases, list):\n raise ValueError(\n f\"Glossary term #{idx} (id={term['id']!r}) in {path}: 'aliases' must be a list, \"\n f\"got {type(aliases).__name__}\"\n )\n for a_idx, alias in enumerate(aliases):\n if not isinstance(alias, str):\n raise ValueError(\n f\"Glossary term #{idx} (id={term['id']!r}) in {path}: alias #{a_idx} \"\n f\"must be a string, got {type(alias).__name__}\"\n )\n if alias == '':\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: alias #{a_idx} is the empty \"\n f\"string. Empty aliases match nothing — remove it.\"\n )\n if alias == term['source']:\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: alias #{a_idx} ({alias!r}) \"\n f\"equals the term's own source. Drop the redundant alias.\"\n )\n if len(set(aliases)) != len(aliases):\n seen = set()\n dup = next((a for a in aliases if a in seen or seen.add(a)), None)\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: alias {dup!r} is duplicated \"\n f\"within the term's aliases list. Each alias must be unique per term.\"\n )\n\n gender = term.get('gender')\n if gender not in VALID_GENDERS:\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: 'gender' must be one of \"\n f\"{list(VALID_GENDERS)}, got {gender!r}\"\n )\n\n confidence = term.get('confidence')\n if confidence not in VALID_CONFIDENCES:\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: 'confidence' must be one of \"\n f\"{list(VALID_CONFIDENCES)}, got {confidence!r}\"\n )\n\n evidence_refs = term.get('evidence_refs', [])\n if not isinstance(evidence_refs, list):\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: 'evidence_refs' must be a list, \"\n f\"got {type(evidence_refs).__name__}\"\n )\n for e_idx, ref in enumerate(evidence_refs):\n if not isinstance(ref, str):\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: evidence_refs #{e_idx} \"\n f\"must be a string, got {type(ref).__name__}\"\n )\n\n notes = term.get('notes', '')\n if not isinstance(notes, str):\n raise ValueError(\n f\"Glossary term id={term['id']!r} in {path}: 'notes' must be a string, \"\n f\"got {type(notes).__name__}\"\n )\n\n\ndef _validate_cross_term_invariants(data, path):\n \"\"\"Term id uniqueness + surface-form uniqueness across terms +\n high_frequency_top_n shape + applied_meta_hashes shape. Run after every\n term has passed individual validation.\"\"\"\n terms = data['terms']\n\n seen_ids = {}\n for idx, t in enumerate(terms):\n if t['id'] in seen_ids:\n raise ValueError(\n f\"Glossary at {path}: duplicate term id {t['id']!r} \"\n f\"(terms #{seen_ids[t['id']]} and #{idx}). Each term must have a unique id.\"\n )\n seen_ids[t['id']] = idx\n\n surface_owner = {} # surface -> (term_id, \"source\"|\"alias\")\n for t in terms:\n surfaces = [(t['source'], 'source')] + [(a, 'alias') for a in t.get('aliases', [])]\n for surface, role in surfaces:\n if surface in surface_owner:\n other_id, other_role = surface_owner[surface]\n raise ValueError(\n f\"Glossary at {path}: surface form {surface!r} appears as \"\n f\"{other_role} of term id={other_id!r} AND as {role} of term id={t['id']!r}. \"\n f\"v2 forbids the same surface form mapping to two different terms — \"\n f\"the term table can only list one translation per word. \"\n f\"Disambiguate by renaming one source, dropping one alias, or merging \"\n f\"the two terms into a single entity.\"\n )\n surface_owner[surface] = (t['id'], role)\n\n if 'high_frequency_top_n' in data:\n top_n_val = data['high_frequency_top_n']\n if isinstance(top_n_val, bool) or not isinstance(top_n_val, int):\n raise ValueError(\n f\"Glossary at {path}: 'high_frequency_top_n' must be an integer, \"\n f\"got {type(top_n_val).__name__}\"\n )\n\n if 'applied_meta_hashes' in data:\n amh = data['applied_meta_hashes']\n if not isinstance(amh, dict):\n raise ValueError(\n f\"Glossary at {path}: 'applied_meta_hashes' must be an object, \"\n f\"got {type(amh).__name__}\"\n )\n for k, v in amh.items():\n if not isinstance(k, str):\n raise ValueError(\n f\"Glossary at {path}: 'applied_meta_hashes' key {k!r} must be a string\"\n )\n if not isinstance(v, str) or not _HEX_RE.match(v):\n raise ValueError(\n f\"Glossary at {path}: 'applied_meta_hashes[{k!r}]' must be a hex string, \"\n f\"got {v!r}\"\n )\n\n\ndef _validate_v1_shape(terms, data, path):\n \"\"\"Pre-upgrade validation for v1 inputs — catches bad source/target/etc.\n before defaults paper over them.\"\"\"\n for i, t in enumerate(terms):\n if not isinstance(t, dict):\n raise ValueError(f\"Glossary term #{i} in {path} must be an object\")\n for required in ('source', 'target'):\n if required not in t:\n raise ValueError(\n f\"Glossary term #{i} in {path} missing required field '{required}'\"\n )\n if not isinstance(t[required], str):\n raise ValueError(\n f\"Glossary term #{i} in {path}: field '{required}' must be a string, \"\n f\"got {type(t[required]).__name__}\"\n )\n if 'category' in t and not isinstance(t['category'], str):\n raise ValueError(\n f\"Glossary term #{i} in {path}: field 'category' must be a string, \"\n f\"got {type(t['category']).__name__}\"\n )\n if 'frequency' in t:\n if isinstance(t['frequency'], bool) or not isinstance(t['frequency'], int):\n raise ValueError(\n f\"Glossary term #{i} in {path}: field 'frequency' must be an integer, \"\n f\"got {type(t['frequency']).__name__}\"\n )\n if 'high_frequency_top_n' in data:\n top_n_val = data['high_frequency_top_n']\n if isinstance(top_n_val, bool) or not isinstance(top_n_val, int):\n raise ValueError(\n f\"Glossary at {path}: 'high_frequency_top_n' must be an integer, \"\n f\"got {type(top_n_val).__name__}\"\n )\n\n\ndef load_glossary(path):\n \"\"\"Load and validate a glossary file. v1 files are auto-upgraded to v2\n on disk (atomic write-back) and a one-line stderr notice is emitted.\n Raises actionable errors on bad input.\"\"\"\n if not os.path.exists(path):\n raise FileNotFoundError(f\"Glossary not found: {path}\")\n\n with open(path, 'r', encoding='utf-8') as f:\n try:\n data = json.load(f)\n except json.JSONDecodeError as e:\n raise ValueError(f\"Glossary at {path} is not valid JSON: {e}\") from e\n\n if not isinstance(data, dict):\n raise ValueError(f\"Glossary at {path} must be a JSON object, got {type(data).__name__}\")\n\n version = data.get('version')\n if version not in (1, 2):\n raise ValueError(\n f\"Glossary schema version mismatch in {path}: \"\n f\"expected 1 or 2, got {version!r}. \"\n f\"Delete the file to rebuild, or migrate it by hand.\"\n )\n\n terms = data.get('terms')\n if not isinstance(terms, list):\n raise ValueError(f\"Glossary at {path} must have a 'terms' array\")\n\n if version == 1:\n _validate_v1_shape(terms, data, path)\n\n # v1 legitimately allows two terms sharing a source (differentiated by\n # category). v2 forbids it because the term table can only list one\n # translation per surface form. Reject the upgrade with disambiguation\n # guidance BEFORE writing anything to disk.\n src_groups = {}\n for idx, t in enumerate(terms):\n src_groups.setdefault(t['source'], []).append(idx)\n for src, idxs in src_groups.items():\n if len(idxs) > 1:\n cats = [terms[i].get('category', '') for i in idxs]\n raise ValueError(\n f\"v1→v2 upgrade aborted for {path}: {len(idxs)} terms share source \"\n f\"{src!r} (categories: {cats}). v2 forbids polysemous surface forms because \"\n f\"the term table can only list one translation per word. Disambiguate by \"\n f\"hand (e.g., change one source to {src!r} + a qualifier such as \"\n f\"'{src} (Inc.)') and re-load. The file on disk has NOT been modified.\"\n )\n\n # Apply defaults, then run strict v2 validation. If v2 validation fails\n # here, the file is NOT rewritten — the user gets the original v1 back.\n data['version'] = GLOSSARY_SCHEMA_VERSION\n for term in terms:\n _v2_term_defaults(term)\n data.setdefault('applied_meta_hashes', {})\n\n for i, t in enumerate(terms):\n _validate_term_strict(t, i, path)\n _validate_cross_term_invariants(data, path)\n\n save_glossary(path, data)\n sys.stderr.write(f\"Upgraded glossary.json at {path} from v1 to v2.\\n\")\n else:\n for i, t in enumerate(terms):\n _validate_term_strict(t, i, path)\n _validate_cross_term_invariants(data, path)\n\n return data\n\n\ndef save_glossary(path, glossary):\n \"\"\"Atomically write the glossary as v2. Re-asserts the v2 invariants\n (per-term shape, surface-form uniqueness, etc.) before writing.\"\"\"\n glossary['version'] = GLOSSARY_SCHEMA_VERSION\n glossary.setdefault('applied_meta_hashes', {})\n for term in glossary.get('terms', []):\n _v2_term_defaults(term)\n\n for i, t in enumerate(glossary.get('terms', [])):\n _validate_term_strict(t, i, path)\n _validate_cross_term_invariants(glossary, path)\n\n dirname = os.path.dirname(os.path.abspath(path)) or '.'\n fd, tmp_path = tempfile.mkstemp(dir=dirname, prefix='.glossary-', suffix='.tmp')\n try:\n with os.fdopen(fd, 'w', encoding='utf-8') as f:\n json.dump(glossary, f, ensure_ascii=False, indent=2, sort_keys=True)\n f.write('\\n')\n os.replace(tmp_path, path)\n except Exception:\n if os.path.exists(tmp_path):\n try:\n os.unlink(tmp_path)\n except OSError:\n pass\n raise\n\n\ndef _count_in_text(source, text):\n \"\"\"Count occurrences of source in text. CJK uses substring; ASCII uses\n boundary-aware regex so 'C++' and '.NET' work and 'cat' doesn't match\n 'category'.\"\"\"\n if not source:\n return 0\n if _contains_cjk(source):\n if len(source) \u003c= 1:\n return 0\n return text.count(source)\n if source.isascii():\n escaped = re.escape(source)\n pattern = rf'(?\u003c!\\w){escaped}(?!\\w)'\n return len(re.findall(pattern, text))\n return text.count(source)\n\n\ndef _appears_in_text(source, text):\n \"\"\"Boundary-aware presence check. Single-CJK-char sources return False\n (would over-match).\"\"\"\n return _count_in_text(source, text) > 0\n\n\ndef _term_appears_in_text(term, text):\n \"\"\"True if the term's source OR any alias appears in text. Each surface\n form is checked via the same boundary-aware matcher.\"\"\"\n if _appears_in_text(term.get('source', ''), text):\n return True\n for alias in term.get('aliases', []):\n if _appears_in_text(alias, text):\n return True\n return False\n\n\ndef count_frequencies(glossary_path, chunks_dir):\n \"\"\"Scan all source chunks and write per-term frequency back into the glossary.\n\n Frequency = sum of source-occurrences plus every alias-occurrence. Source\n chunk discovery excludes `output_chunk*.md` files so re-runs don't\n double-count translated text.\n \"\"\"\n glossary = load_glossary(glossary_path)\n chunks_dir_path = Path(chunks_dir)\n\n chunk_paths = sorted(\n p for p in chunks_dir_path.glob('chunk*.md')\n if not p.name.startswith('output_')\n )\n\n if not chunk_paths:\n print(f\"Warning: no chunk*.md files found under {chunks_dir}\", file=sys.stderr)\n\n all_text_parts = []\n for p in chunk_paths:\n try:\n all_text_parts.append(p.read_text(encoding='utf-8'))\n except OSError as e:\n print(f\"Warning: could not read {p}: {e}\", file=sys.stderr)\n all_text = '\\n'.join(all_text_parts)\n\n for term in glossary['terms']:\n total = 0\n for surface in [term.get('source', '')] + list(term.get('aliases', [])):\n if not surface:\n continue\n if _contains_cjk(surface) and len(surface) \u003c= 1:\n print(\n f\"Warning: skipping frequency count for single-character CJK surface \"\n f\"{surface!r} in term id={term.get('id')!r} (would over-match as substring)\",\n file=sys.stderr,\n )\n continue\n total += _count_in_text(surface, all_text)\n term['frequency'] = total\n\n save_glossary(glossary_path, glossary)\n\n\ndef select_terms_for_chunk(glossary, chunk_text, top_n=None, max_terms=DEFAULT_MAX_TERMS):\n \"\"\"Return terms relevant to a single chunk: union of (terms whose source\n or any alias appears in chunk_text) and (top-N most-frequent terms across\n the whole book).\n\n Local hits are protected — if the union exceeds max_terms, local hits keep\n their slots and global top-N is what gets truncated.\n \"\"\"\n if top_n is None:\n top_n = glossary.get('high_frequency_top_n', DEFAULT_TOP_N)\n\n terms = glossary.get('terms', [])\n if not terms:\n return []\n\n sort_key = lambda t: (-t.get('frequency', 0), t.get('source', ''))\n\n local_terms = [t for t in terms if t.get('source') and _term_appears_in_text(t, chunk_text)]\n local_terms.sort(key=sort_key)\n\n local_ids = {t.get('id', t.get('source')) for t in local_terms}\n top_pool = sorted(\n (t for t in terms if t.get('id', t.get('source')) not in local_ids),\n key=sort_key,\n )\n top_terms = top_pool[:max(0, top_n)]\n\n if max_terms \u003c= 0:\n return []\n if len(local_terms) >= max_terms:\n return local_terms[:max_terms]\n remaining = max_terms - len(local_terms)\n return local_terms + top_terms[:remaining]\n\n\ndef format_terms_for_prompt(terms):\n \"\"\"Render a 3-col markdown table: 原文 | 别名 | 译文.\n\n Empty input yields empty string so the caller can omit the rule line\n entirely. The 别名 column is empty for terms without aliases. Pipes in\n any field escape to \\\\|.\n \"\"\"\n if not terms:\n return ''\n rows = ['| 原文 | 别名 | 译文 |', '|------|------|------|']\n for t in terms:\n source = t.get('source', '').replace('|', '\\\\|')\n aliases = t.get('aliases', []) or []\n alias_str = ', '.join(a.replace('|', '\\\\|') for a in aliases)\n target = t.get('target', '').replace('|', '\\\\|')\n rows.append(f\"| {source} | {alias_str} | {target} |\")\n return '\\n'.join(rows)\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Glossary management for translate-book\")\n sub = parser.add_subparsers(dest='cmd', required=True)\n\n p_count = sub.add_parser('count-frequencies', help=\"Update frequencies in glossary.json\")\n p_count.add_argument('temp_dir', help=\"Path to \u003cbook>_temp/ directory\")\n\n p_print = sub.add_parser('print-terms-for-chunk', help=\"Print markdown table of terms for a chunk\")\n p_print.add_argument('temp_dir')\n p_print.add_argument('chunk_filename', help=\"e.g. chunk0001.md\")\n p_print.add_argument('--top-n', type=int, default=None,\n help=\"Override high_frequency_top_n from glossary.json\")\n p_print.add_argument('--max-terms', type=int, default=DEFAULT_MAX_TERMS,\n help=f\"Cap on terms in the table (default: {DEFAULT_MAX_TERMS})\")\n\n p_hash = sub.add_parser('compute-hash', help=\"Print glossary_hash to stdout\")\n p_hash.add_argument('temp_dir')\n\n args = parser.parse_args()\n glossary_path = os.path.join(args.temp_dir, 'glossary.json')\n\n if args.cmd == 'count-frequencies':\n count_frequencies(glossary_path, args.temp_dir)\n elif args.cmd == 'print-terms-for-chunk':\n glossary = load_glossary(glossary_path)\n chunk_path = os.path.join(args.temp_dir, args.chunk_filename)\n with open(chunk_path, 'r', encoding='utf-8') as f:\n chunk_text = f.read()\n terms = select_terms_for_chunk(\n glossary, chunk_text, top_n=args.top_n, max_terms=args.max_terms\n )\n table = format_terms_for_prompt(terms)\n if table:\n print(table)\n elif args.cmd == 'compute-hash':\n glossary = load_glossary(glossary_path)\n print(glossary_hash(glossary))\n\n\nif __name__ == '__main__':\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":22286,"content_sha256":"a2cac92ef8ad63b2c8f4a99063ae2baf07a90e62ff70f86d04504b8bbd89efec"},{"filename":"scripts/manifest.py","content":"#!/usr/bin/env python3\n\"\"\"\nmanifest.py - Manifest management for chunk tracking and merge validation.\n\"\"\"\n\nimport os\nimport json\nimport hashlib\n\n\ndef file_hash(filepath):\n \"\"\"Compute SHA-256 hash of a file.\"\"\"\n h = hashlib.sha256()\n with open(filepath, 'rb') as f:\n for block in iter(lambda: f.read(8192), b''):\n h.update(block)\n return h.hexdigest()\n\n\ndef read_output_text(filepath):\n \"\"\"Read a translated output chunk as UTF-8 text.\n\n Returns None when the file cannot be read or decoded. Callers treat None\n the same as blank content: the chunk has no usable translation.\n \"\"\"\n try:\n with open(filepath, 'r', encoding='utf-8') as f:\n return f.read()\n except (OSError, UnicodeDecodeError):\n return None\n\n\ndef create_manifest(temp_dir, chunk_files, source_md_path):\n \"\"\"Create manifest.json after splitting.\n\n Args:\n temp_dir: temp directory path\n chunk_files: list of chunk filenames (e.g. ['chunk0001.md', ...])\n source_md_path: path to the source input.md\n \"\"\"\n source_hash = file_hash(source_md_path) if os.path.exists(source_md_path) else \"\"\n\n chunks = []\n for order, filename in enumerate(chunk_files, 1):\n filepath = os.path.join(temp_dir, filename)\n # Derive output filename: chunk0001.md -> output_chunk0001.md\n output_filename = f\"output_{filename}\"\n chunk_id = os.path.splitext(filename)[0] # e.g. \"chunk0001\"\n\n chunks.append({\n \"id\": chunk_id,\n \"order\": order,\n \"source_file\": filename,\n \"source_hash\": file_hash(filepath) if os.path.exists(filepath) else \"\",\n \"output_file\": output_filename,\n })\n\n manifest = {\n \"chunk_count\": len(chunks),\n \"source_hash\": source_hash,\n \"chunks\": chunks,\n }\n\n manifest_path = os.path.join(temp_dir, \"manifest.json\")\n with open(manifest_path, 'w', encoding='utf-8') as f:\n json.dump(manifest, f, indent=2, ensure_ascii=False)\n\n print(f\"Created manifest.json ({len(chunks)} chunks)\")\n return manifest\n\n\ndef load_manifest(temp_dir):\n \"\"\"Load manifest.json from temp_dir. Returns None if not found.\"\"\"\n manifest_path = os.path.join(temp_dir, \"manifest.json\")\n if not os.path.exists(manifest_path):\n return None\n with open(manifest_path, 'r', encoding='utf-8') as f:\n return json.load(f)\n\n\ndef validate_for_merge(temp_dir):\n \"\"\"Validate that all chunks have been translated before merging.\n\n Returns (ok, ordered_output_files, warnings) where:\n ok: True if merge can proceed\n ordered_output_files: list of output file paths in order\n warnings: list of warning strings\n \"\"\"\n manifest = load_manifest(temp_dir)\n if manifest is None:\n # No manifest — fall back to legacy glob-based merge\n return True, None, [\"No manifest.json found, using legacy merge\"]\n\n errors = []\n warnings = []\n ordered_output_files = []\n\n for chunk in sorted(manifest[\"chunks\"], key=lambda c: c[\"order\"]):\n output_path = os.path.join(temp_dir, chunk[\"output_file\"])\n source_path = os.path.join(temp_dir, chunk[\"source_file\"])\n\n # Check source file exists — reject outputs without source chunks\n if not os.path.exists(source_path):\n errors.append(\n f\"Missing source: {chunk['source_file']} (chunk {chunk['id']}) — \"\n f\"cannot verify output integrity without source chunk\"\n )\n continue\n\n # Check source hash matches — detect stale outputs from changed sources\n if chunk.get(\"source_hash\"):\n current_hash = file_hash(source_path)\n if current_hash != chunk[\"source_hash\"]:\n errors.append(\n f\"Source changed since splitting: {chunk['source_file']} \"\n f\"(chunk {chunk['id']}). \"\n f\"Expected hash {chunk['source_hash'][:12]}..., \"\n f\"got {current_hash[:12]}... — \"\n f\"delete output and re-translate, or re-run convert.py to re-split\"\n )\n continue\n\n # Check output exists\n if not os.path.exists(output_path):\n errors.append(f\"Missing output: {chunk['output_file']} (chunk {chunk['id']})\")\n continue\n\n # Check non-empty. Whitespace-only files have bytes on disk but merge\n # to nothing after strip(), silently dropping the chunk's content —\n # treat them exactly like empty files.\n output_size = os.path.getsize(output_path)\n if output_size == 0:\n errors.append(f\"Empty output: {chunk['output_file']} (chunk {chunk['id']})\")\n continue\n output_text = read_output_text(output_path)\n if output_text is None:\n errors.append(\n f\"Unreadable output: {chunk['output_file']} (chunk {chunk['id']}) — \"\n f\"not valid UTF-8 text\"\n )\n continue\n if not output_text.strip():\n errors.append(\n f\"Blank output: {chunk['output_file']} (chunk {chunk['id']}) — \"\n f\"whitespace-only content would be silently dropped on merge\"\n )\n continue\n\n # Check abnormally short\n if os.path.exists(source_path):\n source_size = os.path.getsize(source_path)\n if source_size > 0 and output_size \u003c source_size * 0.1:\n warnings.append(\n f\"Suspiciously short: {chunk['output_file']} \"\n f\"({output_size} bytes vs source {source_size} bytes)\"\n )\n\n ordered_output_files.append(output_path)\n\n if errors:\n for e in errors:\n print(f\"ERROR: {e}\")\n return False, None, warnings\n\n for w in warnings:\n print(f\"WARNING: {w}\")\n\n return True, ordered_output_files, warnings\n","content_type":"text/x-python; charset=utf-8","language":"python","size":5943,"content_sha256":"880219f31cc1e4b50fd82dc3fe6d7cd0c6de9446daf18659b5264ce3c8c316b5"},{"filename":"scripts/merge_and_build.py","content":"#!/usr/bin/env python3\n\"\"\"\nmerge_and_build.py - Merge translated pages and build final outputs\nCombines original steps 4-7: merge -> HTML -> TOC -> DOCX/EPUB/PDF\n\nUsage: merge_and_build.py --temp-dir \u003cpath> [--title \u003ctitle>] [--author \u003cauthor>] [--lang \u003clang>]\n\"\"\"\n\nimport os\nimport sys\nimport re\nimport glob\nimport shutil\nimport subprocess\nimport argparse\nimport html as _html_lib\nfrom collections import Counter\nfrom html.parser import HTMLParser\nfrom pathlib import Path\n\nfrom manifest import read_output_text, validate_for_merge\n\n\n# =============================================================================\n# Image structure validation helpers\n# =============================================================================\n\n# Markdown image: `![alt](url)` or `![alt](url \"title\")`.\n# - Negative lookbehind on `\\` skips escaped `\\![...]` (renders as literal text).\n# - Closing `)` is required — a missing `)` means the image won't render, so\n# such a fragment must NOT count as a preserved image reference.\n_MD_IMG_RE = re.compile(r'(?\u003c!\\\\)!\\[[^\\]]*\\]\$\\s*([^)\\s]+)[^)]*\$')\n_VALID_ATTR_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_:.\\-]*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

)\n\n\nclass _ImgTagCollector(HTMLParser):\n \"\"\"Collects every \u003cimg> tag found in fed text. Uses stdlib HTMLParser, which\n correctly handles `>` inside quoted attribute values — unlike a plain\n `\u003cimg\\\\b[^>]*>` regex, which would truncate at the first `>`.\"\"\"\n\n def __init__(self):\n super().__init__(convert_charrefs=True)\n self.records = [] # list of (raw_tag_text, attrs_list)\n\n def handle_starttag(self, tag, attrs):\n if tag == 'img':\n self.records.append((self.get_starttag_text(), list(attrs)))\n\n handle_startendtag = handle_starttag\n\n\ndef _scan_img_tags(text):\n \"\"\"Return (Counter of \u003cimg> srcs, list of (raw_tag, bad_attr_name) tuples).\n\n Feeds the entire text to HTMLParser rather than pre-extracting tags via regex,\n so quoted attribute values containing `>` are handled correctly.\"\"\"\n src_counts = Counter()\n bad_attrs = []\n parser = _ImgTagCollector()\n try:\n parser.feed(text)\n parser.close()\n except Exception as e:\n bad_attrs.append(('\u003cunparseable input>', f'\u003cparser error: {e}>'))\n return src_counts, bad_attrs\n for raw_tag, attrs in parser.records:\n for name, _ in attrs:\n if not _VALID_ATTR_NAME_RE.match(name):\n bad_attrs.append((raw_tag, name))\n for name, val in attrs:\n if name == 'src' and val:\n src_counts[val] += 1\n return src_counts, bad_attrs\n\n\ndef _scan_image_refs(text):\n \"\"\"Return (Counter html_srcs, Counter md_srcs, list bad_attrs).\"\"\"\n html_srcs, bad_attrs = _scan_img_tags(text)\n md_srcs = Counter(_MD_IMG_RE.findall(text))\n return html_srcs, md_srcs, bad_attrs\n\n\ndef _validate_chunk_images(temp_dir):\n \"\"\"Verify each output_chunk*.md preserves the image structure of its chunk*.md.\n\n Bad-attribute detection uses a per-chunk DELTA: a malformed \u003cimg> attribute\n is flagged only if it appears in the output chunk but not in the source\n chunk. This avoids false positives on code blocks that legitimately contain\n deliberately-broken \u003cimg> examples — both chunks carry the same example, so\n the delta is empty.\n\n Returns False on any divergence; collects all errors and prints them\n together so an agent can fix many chunks in one pass.\n \"\"\"\n temp_path = Path(temp_dir)\n errors = []\n for src_chunk in sorted(temp_path.glob('chunk*.md')):\n if src_chunk.name.startswith('output_'):\n continue\n out_chunk = temp_path / f'output_{src_chunk.name}'\n if not out_chunk.exists():\n continue # missing-output is the manifest validator's job\n src_html, src_md, src_bad = _scan_image_refs(src_chunk.read_text(encoding='utf-8'))\n out_html, out_md, out_bad = _scan_image_refs(out_chunk.read_text(encoding='utf-8'))\n\n src_bad_counts = Counter(name for _, name in src_bad)\n out_bad_counts = Counter(name for _, name in out_bad)\n new_bad_counts = out_bad_counts - src_bad_counts\n if new_bad_counts:\n new_bad_examples = [\n (raw_tag, attr_name)\n for raw_tag, attr_name in out_bad\n if new_bad_counts.get(attr_name, 0) > 0\n ]\n for raw_tag, attr_name in new_bad_examples:\n errors.append(\n f\"ERROR: {out_chunk.name} introduced malformed \u003cimg> tag (not present in source)\\n\"\n f\" tag: {raw_tag}\\n\"\n f\" problem: attribute name '{attr_name}' is not a valid HTML identifier\\n\"\n f\" likely cause: an unescaped quote inside alt=\\\"...\\\" or title=\\\"...\\\" closed the attribute early\\n\"\n f\" fix: in {out_chunk.name}, replace the inner quote with a curly quote in the target language or with " / '\\n\"\n f\" source chunk for reference: {src_chunk.name}\"\n )\n\n if src_html != out_html or src_md != out_md:\n errors.append(\n f\"ERROR: {out_chunk.name} image references diverge from {src_chunk.name}\\n\"\n f\" missing \u003cimg src> (count): {sorted((src_html - out_html).items()) or 'none'}\\n\"\n f\" extra \u003cimg src> (count): {sorted((out_html - src_html).items()) or 'none'}\\n\"\n f\" missing ![](path) (count): {sorted((src_md - out_md).items()) or 'none'}\\n\"\n f\" extra ![](path) (count): {sorted((out_md - src_md).items()) or 'none'}\\n\"\n f\" fix: restore the missing image refs in {out_chunk.name} from {src_chunk.name}\"\n )\n\n if errors:\n print(\"\\n=== Image validation failed ===\")\n for e in errors:\n print(e)\n print()\n return False\n return True\n\n\n# Pandoc footnote labels cannot contain whitespace, ^, [ or ]; this also keeps\n# `[^1^]` pseudo-footnotes from older conversions out of the comparison.\n_FOOTNOTE_LABEL_RE = re.compile(r'\\[\\^([^\\]\\s^\\[]+)\\]')\n\n\ndef _scan_footnote_labels(text):\n \"\"\"Return (Counter of reference labels, Counter of definition labels).\"\"\"\n refs, defs = Counter(), Counter()\n for m in _FOOTNOTE_LABEL_RE.finditer(text):\n line_start = text.rfind('\\n', 0, m.start()) + 1\n indent = text[line_start:m.start()]\n is_definition = len(indent) \u003c= 3 and not indent.strip() and text.startswith(':', m.end())\n (defs if is_definition else refs)[m.group(1)] += 1\n return refs, defs\n\n\ndef _validate_chunk_footnotes(temp_dir):\n \"\"\"Verify each output_chunk*.md keeps the footnote labels of its chunk*.md.\n\n A dropped or renumbered `[^N]` / `[^N]:` leaves a dangling reference or an\n orphaned note in the merged book. Returns False on any divergence and\n prints every error at once.\n \"\"\"\n temp_path = Path(temp_dir)\n errors = []\n for src_chunk in sorted(temp_path.glob('chunk*.md')):\n out_chunk = temp_path / f'output_{src_chunk.name}'\n if not out_chunk.exists():\n continue\n src_text = read_output_text(src_chunk)\n out_text = read_output_text(out_chunk)\n if src_text is None or out_text is None:\n continue # unreadable files are the manifest validator's job\n src_refs, src_defs = _scan_footnote_labels(src_text)\n out_refs, out_defs = _scan_footnote_labels(out_text)\n if src_refs != out_refs or src_defs != out_defs:\n errors.append(\n f\"ERROR: {out_chunk.name} footnote markers diverge from {src_chunk.name}\\n\"\n f\" missing references [^N] (count): {sorted((src_refs - out_refs).items()) or 'none'}\\n\"\n f\" extra references [^N] (count): {sorted((out_refs - src_refs).items()) or 'none'}\\n\"\n f\" missing definitions [^N]: (count): {sorted((src_defs - out_defs).items()) or 'none'}\\n\"\n f\" extra definitions [^N]: (count): {sorted((out_defs - src_defs).items()) or 'none'}\\n\"\n f\" fix: in {out_chunk.name}, restore every footnote marker exactly as in {src_chunk.name}; \"\n f\"translate only the note text\"\n )\n\n if errors:\n print(\"\\n=== Footnote validation failed ===\")\n for e in errors:\n print(e)\n print()\n return False\n return True\n\n\ndef _check_generated_html_sanity(html_path):\n \"\"\"Sanity-check generated HTML for malformed \u003cimg> tags. Returns False on problems.\n\n Note: we deliberately do NOT flag `<img` in the rendered HTML — books that\n discuss HTML in prose or code blocks legitimately render escaped `\u003cimg>` text,\n and that's not a corruption signal. Real corruption produces a malformed\n actual `\u003cimg>` tag, which the attribute-name check catches.\"\"\"\n try:\n text = Path(html_path).read_text(encoding='utf-8')\n except Exception as e:\n print(f\"ERROR: cannot read {html_path}: {e}\")\n return False\n\n _, bad_attrs = _scan_img_tags(text)\n if not bad_attrs:\n return True\n\n print(f\"ERROR: image sanity check failed on {Path(html_path).name}\")\n for raw_tag, attr_name in bad_attrs:\n print(f\" - malformed \u003cimg>: {raw_tag}\")\n print(f\" bad attribute name: '{attr_name}'\")\n print(\n \" fix: inspect output.md and the corresponding output_chunk*.md;\\n\"\n \" if alt text contains literal quotes, replace with curly quotes or HTML entity\"\n )\n return False\n\n# Try to import BeautifulSoup for TOC generation\ntry:\n from bs4 import BeautifulSoup\n BS4_AVAILABLE = True\nexcept ImportError:\n BS4_AVAILABLE = False\n\n# Try to import markdown\ntry:\n import markdown\n MARKDOWN_AVAILABLE = True\nexcept ImportError:\n MARKDOWN_AVAILABLE = False\n\nSCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))\n\n# =============================================================================\n# Language configuration — single source of truth for lang-dependent values\n# =============================================================================\n\nLANG_CONFIG = {\n 'zh': {\n 'lang_attr': 'zh-CN',\n 'font_family': \"'FangSong', '仿宋', 'STFangSong', '华文仿宋', serif\",\n 'font_family_ebook': '\"FangSong\", \"FangSong_GB2312\", \"仿宋\", \"仿宋_GB2312\", \"STFangSong\", \"SimSun\", serif',\n 'toc_label': '目录',\n 'pdf_font': 'FangSong',\n },\n 'en': {\n 'lang_attr': 'en',\n 'font_family': \"Georgia, 'Times New Roman', Times, serif\",\n 'font_family_ebook': 'Georgia, \"Times New Roman\", Times, serif',\n 'toc_label': 'Contents',\n 'pdf_font': 'Georgia',\n },\n 'ja': {\n 'lang_attr': 'ja',\n 'font_family': \"'Hiragino Mincho ProN', 'Yu Mincho', 'MS Mincho', serif\",\n 'font_family_ebook': '\"Hiragino Mincho ProN\", \"Yu Mincho\", \"MS Mincho\", serif',\n 'toc_label': '目次',\n 'pdf_font': 'Hiragino Mincho ProN',\n },\n 'ko': {\n 'lang_attr': 'ko',\n 'font_family': \"'Nanum Myeongjo', 'Batang', serif\",\n 'font_family_ebook': '\"Nanum Myeongjo\", \"Batang\", serif',\n 'toc_label': '목차',\n 'pdf_font': 'Nanum Myeongjo',\n },\n 'fr': {\n 'lang_attr': 'fr',\n 'font_family': \"Georgia, 'Times New Roman', Times, serif\",\n 'font_family_ebook': 'Georgia, \"Times New Roman\", Times, serif',\n 'toc_label': 'Table des matières',\n 'pdf_font': 'Georgia',\n },\n 'de': {\n 'lang_attr': 'de',\n 'font_family': \"Georgia, 'Times New Roman', Times, serif\",\n 'font_family_ebook': 'Georgia, \"Times New Roman\", Times, serif',\n 'toc_label': 'Inhaltsverzeichnis',\n 'pdf_font': 'Georgia',\n },\n 'es': {\n 'lang_attr': 'es',\n 'font_family': \"Georgia, 'Times New Roman', Times, serif\",\n 'font_family_ebook': 'Georgia, \"Times New Roman\", Times, serif',\n 'toc_label': 'Índice',\n 'pdf_font': 'Georgia',\n },\n}\n\n# Default fallback for unknown languages\n_DEFAULT_LANG_CONFIG = {\n 'lang_attr': 'en',\n 'font_family': \"Georgia, 'Times New Roman', Times, serif\",\n 'font_family_ebook': 'Georgia, \"Times New Roman\", Times, serif',\n 'toc_label': 'Contents',\n 'pdf_font': 'Georgia',\n}\n\n\ndef get_lang_config(lang_code):\n \"\"\"Get language config, falling back to defaults for unknown languages.\"\"\"\n return LANG_CONFIG.get(lang_code, _DEFAULT_LANG_CONFIG)\n\n\ndef load_config(temp_dir):\n \"\"\"Load configuration from config.txt\"\"\"\n config_file = os.path.join(temp_dir, 'config.txt')\n if not os.path.exists(config_file):\n print(\"Error: config.txt not found in temp directory.\")\n sys.exit(1)\n\n config = {}\n with open(config_file, 'r', encoding='utf-8') as f:\n for line in f:\n if '=' in line and not line.strip().startswith('#'):\n key, value = line.strip().split('=', 1)\n config[key] = value\n return config\n\n\ndef natural_sort_key(text):\n \"\"\"Natural sorting key for filenames with numbers\"\"\"\n return [int(t) if t.isdigit() else t.lower() for t in re.split(r'(\\d+)', text)]\n\n\n# =============================================================================\n# Step 4: Merge translated markdown files\n# =============================================================================\n\ndef merge_markdown_files(temp_dir):\n \"\"\"Merge all translated output files into output.md\"\"\"\n print(\"=== Merging translated markdown files ===\")\n\n output_md = os.path.join(temp_dir, 'output.md')\n\n # Always validate manifest, even if output.md exists (catch stale/corrupt outputs)\n ok, ordered_files, warnings = validate_for_merge(temp_dir)\n\n # Image and footnote validation run unconditionally — bad chunks invalidate any cached output.md\n images_ok = _validate_chunk_images(temp_dir)\n footnotes_ok = _validate_chunk_footnotes(temp_dir)\n if not (images_ok and footnotes_ok):\n if os.path.exists(output_md):\n print(\"Removing stale output.md (built from chunks that failed validation)\")\n os.remove(output_md)\n return False\n\n if os.path.exists(output_md):\n if not ok:\n print(f\"WARNING: output.md exists but manifest validation failed — deleting stale output.md\")\n os.remove(output_md)\n else:\n # Check if any output_chunk is newer than output.md (re-translated chunks)\n output_md_mtime = os.path.getmtime(output_md)\n newer_chunks = []\n if ordered_files:\n newer_chunks = [\n os.path.basename(f) for f in ordered_files\n if os.path.getmtime(f) > output_md_mtime\n ]\n if newer_chunks:\n print(f\"Re-merging — {len(newer_chunks)} chunk(s) newer than output.md: {', '.join(newer_chunks[:5])}{'...' if len(newer_chunks) > 5 else ''}\")\n os.remove(output_md)\n else:\n print(f\"Skipping merge - output.md already exists and is up to date\")\n return True\n\n if not ok:\n print(\"ERROR: Merge validation failed. Fix the issues above before merging.\")\n return False\n\n if ordered_files is not None:\n # Manifest-based merge\n print(f\"Merging {len(ordered_files)} translated files (manifest-ordered)\")\n merged_content = \"\"\n for file_path in ordered_files:\n content = read_output_text(file_path)\n if content is None:\n print(f\"ERROR: Cannot read {os.path.basename(file_path)} — aborting merge\")\n return False\n content = content.strip()\n if not content:\n # validate_for_merge already rejects blank outputs; this is a\n # last line of defense so a chunk can never vanish silently.\n print(f\"ERROR: Blank output {os.path.basename(file_path)} — aborting merge\")\n return False\n merged_content += content + \"\\n\\n\"\n else:\n # Legacy fallback: glob-based merge (no manifest)\n print(\"WARNING: No manifest.json found — using legacy glob-based merge.\")\n print(\" For hash validation, re-run convert.py to generate manifest.json\")\n\n # Match chunk output files\n output_files = glob.glob(os.path.join(temp_dir, 'output_chunk*.md'))\n\n # Count original source files\n original_files = glob.glob(os.path.join(temp_dir, 'chunk*.md'))\n original_files = [f for f in original_files if not os.path.basename(f).startswith('output_')]\n\n if not output_files:\n print(\"Error: No translated markdown files found.\")\n return False\n\n # Build expected output filename for each source file and verify 1:1 match\n source_basenames = sorted(\n [os.path.basename(f) for f in original_files],\n key=natural_sort_key\n )\n expected_outputs = set(f\"output_{name}\" for name in source_basenames)\n actual_outputs = set(os.path.basename(f) for f in output_files)\n\n missing = expected_outputs - actual_outputs\n orphaned = actual_outputs - expected_outputs\n\n if missing or orphaned:\n if missing:\n print(f\"ERROR: Missing translations for: {', '.join(sorted(missing, key=natural_sort_key))}\")\n if orphaned:\n print(f\"ERROR: Orphaned outputs (no matching source): {', '.join(sorted(orphaned, key=natural_sort_key))}\")\n return False\n\n # Verify no empty, unreadable, or whitespace-only output files\n for fp in output_files:\n if os.path.getsize(fp) == 0:\n print(f\"ERROR: Empty output file: {os.path.basename(fp)}\")\n return False\n text = read_output_text(fp)\n if text is None:\n print(f\"ERROR: Unreadable output file: {os.path.basename(fp)}\")\n return False\n if not text.strip():\n print(f\"ERROR: Blank output file: {os.path.basename(fp)}\")\n return False\n\n # Use source order to determine merge order (via expected output names)\n output_files = [\n os.path.join(temp_dir, f\"output_{name}\")\n for name in source_basenames\n ]\n print(f\"Merging {len(output_files)} translated files (legacy glob)\")\n\n merged_content = \"\"\n for file_path in output_files:\n content = read_output_text(file_path)\n if content is None:\n print(f\"ERROR: Cannot read {os.path.basename(file_path)} — aborting merge\")\n return False\n content = content.strip()\n if not content:\n print(f\"ERROR: Blank output {os.path.basename(file_path)} — aborting merge\")\n return False\n merged_content += content + \"\\n\\n\"\n\n try:\n with open(output_md, 'w', encoding='utf-8') as f:\n f.write(merged_content)\n file_size = os.path.getsize(output_md)\n print(f\"Merged into output.md ({file_size:,} bytes)\")\n return True\n except Exception as e:\n print(f\"Error saving merged file: {e}\")\n return False\n\n\n# =============================================================================\n# Step 5: Convert markdown to HTML\n# =============================================================================\n\ndef check_pandoc_available():\n \"\"\"Check if pandoc is available\"\"\"\n try:\n subprocess.run(['pandoc', '--version'], capture_output=True, check=True)\n return True\n except (subprocess.CalledProcessError, FileNotFoundError):\n return False\n\n\ndef convert_with_pandoc(md_file, html_file, title, lang_attr):\n \"\"\"Convert markdown to HTML using pandoc\"\"\"\n cmd = [\n 'pandoc', md_file, '-o', html_file,\n '--standalone',\n '--metadata', f'title={title}',\n '--metadata', f'lang={lang_attr}',\n '--from', 'markdown+smart+east_asian_line_breaks',\n '--to', 'html5',\n # Without this, pandoc emits any TeX it cannot approximate in plain\n # HTML (fractions, roots, display math) as raw `$... translate-book — Skillopedia text.\n '--mathml'\n ]\n\n try:\n subprocess.run(cmd, capture_output=True, text=True, check=True)\n print(\"Converted with pandoc\")\n return True\n except subprocess.CalledProcessError as e:\n print(f\"Pandoc conversion failed: {e.stderr}\")\n return False\n\n\ndef convert_with_python_markdown(md_file, html_file, title):\n \"\"\"Convert markdown to HTML using python-markdown (fallback 1)\"\"\"\n if not MARKDOWN_AVAILABLE:\n return False\n\n try:\n with open(md_file, 'r', encoding='utf-8') as f:\n md_content = f.read()\n\n extensions = ['toc', 'tables', 'fenced_code', 'codehilite', 'nl2br', 'footnotes']\n md = markdown.Markdown(extensions=extensions)\n html_content = md.convert(md_content)\n\n full_html = f\"\"\"\u003c!DOCTYPE html>\n\u003chtml>\n\u003chead>\n\u003cmeta charset=\"UTF-8\">\n\u003ctitle>{title}\u003c/title>\n\u003c/head>\n\u003cbody>\n{html_content}\n\u003c/body>\n\u003c/html>\"\"\"\n\n with open(html_file, 'w', encoding='utf-8') as f:\n f.write(full_html)\n\n print(\"Converted with python-markdown (fallback)\")\n return True\n except Exception as e:\n print(f\"python-markdown conversion failed: {e}\")\n return False\n\n\ndef convert_with_basic_regex(md_file, html_file, title):\n \"\"\"Convert markdown to HTML using basic regex (fallback 2)\"\"\"\n try:\n with open(md_file, 'r', encoding='utf-8') as f:\n md_content = f.read()\n\n html_content = md_content\n\n # Headers\n html_content = re.sub(r'^#### (.*?)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, r'\u003ch4>\\1\u003c/h4>', html_content, flags=re.MULTILINE)\n html_content = re.sub(r'^### (.*?)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, r'\u003ch3>\\1\u003c/h3>', html_content, flags=re.MULTILINE)\n html_content = re.sub(r'^## (.*?)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, r'\u003ch2>\\1\u003c/h2>', html_content, flags=re.MULTILINE)\n html_content = re.sub(r'^# (.*?)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, r'\u003ch1>\\1\u003c/h1>', html_content, flags=re.MULTILINE)\n\n # Bold and italic\n html_content = re.sub(r'\\*\\*(.*?)\\*\\*', r'\u003cstrong>\\1\u003c/strong>', html_content)\n html_content = re.sub(r'\\*(.*?)\\*', r'\u003cem>\\1\u003c/em>', html_content)\n html_content = re.sub(r'_(.*?)_', r'\u003cem>\\1\u003c/em>', html_content)\n\n # Images — escape alt and src so quotes in alt text don't break the tag\n def _md_img_to_html(m):\n alt = _html_lib.escape(m.group(1), quote=True)\n src = _html_lib.escape(m.group(2), quote=True)\n return f'\u003cimg src=\"{src}\" alt=\"{alt}\">'\n html_content = re.sub(r'!\\[([^\\]]*)\\]\$([^)]*)\$', _md_img_to_html, html_content)\n\n # Links\n html_content = re.sub(r'\\[([^\\]]*)\\]\$([^)]*)\$', r'\u003ca href=\"\\2\">\\1\u003c/a>', html_content)\n\n # Lists and paragraphs\n lines = html_content.split('\\n')\n result_lines = []\n in_list = False\n\n for line in lines:\n stripped = line.strip()\n if stripped.startswith('- '):\n if not in_list:\n result_lines.append('\u003cul>')\n in_list = 'ul'\n item = stripped[2:]\n result_lines.append(f'\u003cli>{item}\u003c/li>')\n elif re.match(r'^\\d+\\. ', stripped):\n if not in_list:\n result_lines.append('\u003col>')\n in_list = 'ol'\n item = re.sub(r'^\\d+\\. ', '', stripped)\n result_lines.append(f'\u003cli>{item}\u003c/li>')\n else:\n if in_list:\n result_lines.append(f'\u003c/{in_list}>')\n in_list = False\n if stripped and not stripped.startswith('\u003c'):\n result_lines.append(f'\u003cp>{line}\u003c/p>')\n else:\n result_lines.append(line)\n\n if in_list:\n result_lines.append(f'\u003c/{in_list}>')\n\n html_content = '\\n'.join(result_lines)\n\n # Page separators\n html_content = re.sub(r'\u003cp>---\u003c/p>', '\u003cdiv class=\"page-separator\">\u003c/div>', html_content)\n\n full_html = f\"\"\"\u003c!DOCTYPE html>\n\u003chtml>\n\u003chead>\n\u003cmeta charset=\"UTF-8\">\n\u003ctitle>{title}\u003c/title>\n\u003c/head>\n\u003cbody>\n{html_content}\n\u003c/body>\n\u003c/html>\"\"\"\n\n with open(html_file, 'w', encoding='utf-8') as f:\n f.write(full_html)\n\n print(\"Converted with basic regex (fallback 2)\")\n return True\n except Exception as e:\n print(f\"Basic regex conversion failed: {e}\")\n return False\n\n\ndef apply_template_to_html(html_content, template_file, output_file, title, lang_cfg, author=None):\n \"\"\"Apply a template to HTML content with language-aware substitutions\"\"\"\n if not template_file or not os.path.exists(template_file):\n print(f\"Warning: Template {template_file} not found\")\n return False\n\n try:\n with open(template_file, 'r', encoding='utf-8') as f:\n template_content = f.read()\n\n if '$body

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

in template_content:\n full_html = template_content.replace('$body

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, html_content)\n elif '{{content}}' in template_content:\n full_html = template_content.replace('{{content}}', html_content)\n else:\n if '\u003c/body>' in template_content:\n full_html = template_content.replace('\u003c/body>', f'{html_content}\\n\u003c/body>')\n else:\n full_html = template_content + html_content\n\n # Replace all template placeholders\n full_html = full_html.replace('$title

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, title)\n full_html = full_html.replace('$lang

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, lang_cfg['lang_attr'])\n full_html = full_html.replace('$body_font

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, lang_cfg['font_family'])\n full_html = full_html.replace('$toc_label

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, lang_cfg['toc_label'])\n\n # Inject author meta tag into \u003chead> so calibre_html_publish.py can extract it\n if author:\n author_meta = f'\u003cmeta name=\"author\" content=\"{author}\">'\n if '\u003chead>' in full_html or '\u003chead ' in full_html:\n full_html = re.sub(\n r'(\u003chead[^>]*>)',\n r'\\1\\n ' + author_meta,\n full_html,\n count=1,\n flags=re.IGNORECASE\n )\n\n with open(output_file, 'w', encoding='utf-8') as f:\n f.write(full_html)\n return True\n except Exception as e:\n print(f\"Error applying template: {e}\")\n return False\n\n\ndef process_html_separators(html_file):\n \"\"\"Process page separators in HTML\"\"\"\n try:\n with open(html_file, 'r', encoding='utf-8') as f:\n content = f.read()\n\n content = re.sub(r'\u003chr\\s*/?>', '\u003cdiv class=\"page-separator\">\u003c/div>', content)\n content = re.sub(r'\u003cp>\\s*---\\s*\u003c/p>', '\u003cdiv class=\"page-separator\">\u003c/div>', content)\n\n with open(html_file, 'w', encoding='utf-8') as f:\n f.write(content)\n except Exception as e:\n print(f\"Error processing separators: {e}\")\n\n\ndef convert_md_to_html(temp_dir, title, lang_cfg, author=None):\n \"\"\"Convert output.md to HTML with templates\"\"\"\n print(\"=== Converting markdown to HTML ===\")\n\n md_file = os.path.join(temp_dir, 'output.md')\n if not os.path.exists(md_file):\n print(\"Error: output.md not found.\")\n return False\n\n book_doc_file = os.path.join(temp_dir, 'book_doc.html')\n\n # Skip HTML generation if book_doc.html exists and is newer than output.md\n if os.path.exists(book_doc_file):\n if os.path.getmtime(book_doc_file) > os.path.getmtime(md_file):\n if _check_generated_html_sanity(book_doc_file):\n print(\"Skipping HTML generation - book_doc.html is up to date\")\n return True\n print(\"Stale book_doc.html failed image sanity — regenerating\")\n os.remove(book_doc_file)\n else:\n print(\"Re-generating HTML - output.md is newer\")\n\n temp_html_file = os.path.join(temp_dir, 'output.html')\n\n # Try pandoc -> python-markdown -> basic regex\n success = False\n if check_pandoc_available():\n success = convert_with_pandoc(md_file, temp_html_file, title, lang_cfg['lang_attr'])\n\n if not success:\n success = convert_with_python_markdown(md_file, temp_html_file, title)\n\n if not success:\n success = convert_with_basic_regex(md_file, temp_html_file, title)\n\n if not success:\n print(\"Error: All markdown-to-HTML converters failed\")\n return False\n\n process_html_separators(temp_html_file)\n\n # Extract body content\n try:\n with open(temp_html_file, 'r', encoding='utf-8') as f:\n html_content = f.read()\n except Exception as e:\n print(f\"Error reading HTML file: {e}\")\n return False\n\n body_match = re.search(r'\u003cbody[^>]*>(.*?)\u003c/body>', html_content, re.DOTALL | re.IGNORECASE)\n body_content = body_match.group(1).strip() if body_match else html_content\n\n # Generate book_doc.html with ebook template\n template_ebook = os.path.join(SCRIPT_DIR, 'template_ebook.html')\n book_doc_file = os.path.join(temp_dir, 'book_doc.html')\n apply_template_to_html(body_content, template_ebook, book_doc_file, title, lang_cfg, author)\n\n # Generate book.html with web template\n template_web = os.path.join(SCRIPT_DIR, 'template.html')\n book_file = os.path.join(temp_dir, 'book.html')\n apply_template_to_html(body_content, template_web, book_file, title, lang_cfg, author)\n\n if not _check_generated_html_sanity(book_doc_file):\n return False\n if not _check_generated_html_sanity(book_file):\n return False\n\n print(f\"Generated: output.html, book_doc.html, book.html\")\n return True\n\n\n# =============================================================================\n# Step 6: Add TOC\n# =============================================================================\n\ndef generate_heading_id(text, existing_ids):\n \"\"\"Generate unique ID for heading\"\"\"\n base_id = re.sub(r'[^\\w\\s-]', '', text.lower())\n base_id = re.sub(r'[-\\s]+', '-', base_id)\n base_id = base_id.strip('-')\n\n if not base_id:\n base_id = 'heading'\n\n heading_id = base_id\n counter = 1\n while heading_id in existing_ids:\n heading_id = f\"{base_id}-{counter}\"\n counter += 1\n\n return heading_id\n\n\ndef generate_simple_toc_html(toc_data):\n \"\"\"Generate simple HTML for table of contents\"\"\"\n if not toc_data:\n return \"\"\n\n toc_html = '\u003cul>\\n'\n current_level = 1\n\n for item in toc_data:\n level = item['level']\n text = item['text']\n heading_id = item['id']\n\n if level > current_level:\n while current_level \u003c level:\n toc_html += '\u003cli>\u003cul>\\n'\n current_level += 1\n elif level \u003c current_level:\n while current_level > level:\n toc_html += '\u003c/ul>\u003c/li>\\n'\n current_level -= 1\n\n toc_html += f'\u003cli>\u003ca href=\"#{heading_id}\">{text}\u003c/a>\u003c/li>\\n'\n\n while current_level > 1:\n toc_html += '\u003c/ul>\u003c/li>\\n'\n current_level -= 1\n\n toc_html += '\u003c/ul>\\n'\n return toc_html\n\n\ndef insert_toc_with_bs4(html_file):\n \"\"\"Insert TOC using BeautifulSoup\"\"\"\n try:\n with open(html_file, 'r', encoding='utf-8') as f:\n html_content = f.read()\n except Exception as e:\n print(f\"Error reading HTML file: {e}\")\n return False\n\n soup = BeautifulSoup(html_content, 'html.parser')\n\n toc_data = []\n existing_ids = []\n\n for heading in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):\n level = int(heading.name[1])\n text = heading.get_text().strip()\n if not text:\n continue\n\n heading_id = generate_heading_id(text, existing_ids)\n existing_ids.append(heading_id)\n heading['id'] = heading_id\n toc_data.append({'level': level, 'text': text, 'id': heading_id})\n\n if not toc_data:\n print(\"No headings found for TOC\")\n return False\n\n toc_html = generate_simple_toc_html(toc_data)\n\n toc_content_div = soup.find('div', class_='toc-content')\n if toc_content_div:\n toc_content_div.clear()\n toc_soup = BeautifulSoup(toc_html, 'html.parser')\n toc_content_div.append(toc_soup)\n print(f\"TOC inserted ({len(toc_data)} headings)\")\n else:\n print(\"Warning: .toc-content div not found, TOC not inserted\")\n return False\n\n try:\n with open(html_file, 'w', encoding='utf-8') as f:\n f.write(str(soup))\n return True\n except Exception as e:\n print(f\"Error saving HTML file: {e}\")\n return False\n\n\ndef insert_toc_with_regex(html_file):\n \"\"\"Insert TOC using regex (fallback)\"\"\"\n try:\n with open(html_file, 'r', encoding='utf-8') as f:\n html_content = f.read()\n except Exception as e:\n print(f\"Error reading HTML file: {e}\")\n return False\n\n heading_pattern = r'\u003c(h[1-6])(?:[^>]*)>(.*?)\u003c/\\1>'\n headings = re.findall(heading_pattern, html_content, re.IGNORECASE | re.DOTALL)\n\n if not headings:\n print(\"No headings found for TOC\")\n return False\n\n toc_html = '\u003cul>\\n'\n for i, (tag, text) in enumerate(headings):\n level = int(tag[1])\n clean_text = re.sub(r'\u003c[^>]+>', '', text).strip()\n heading_id = f\"heading-{i+1}\"\n\n old_heading = f'\u003c{tag}>{text}\u003c/{tag}>'\n new_heading = f'\u003c{tag} id=\"{heading_id}\">{text}\u003c/{tag}>'\n html_content = html_content.replace(old_heading, new_heading, 1)\n\n if level > 1:\n for _ in range(level - 1):\n toc_html += ' '\n toc_html += f'\u003cli>\u003ca href=\"#{heading_id}\">{clean_text}\u003c/a>\u003c/li>\\n'\n\n toc_html += '\u003c/ul>\\n'\n\n toc_content_pattern = r'(\u003cdiv[^>]*class=\"toc-content[^\"]*\"[^>]*>).*?(\u003c/div>)'\n if re.search(toc_content_pattern, html_content, re.DOTALL):\n html_content = re.sub(\n toc_content_pattern,\n r'\\1' + toc_html + r'\\2',\n html_content,\n flags=re.DOTALL\n )\n print(f\"TOC inserted ({len(headings)} headings)\")\n else:\n print(\"Warning: .toc-content div not found\")\n return False\n\n try:\n with open(html_file, 'w', encoding='utf-8') as f:\n f.write(html_content)\n return True\n except Exception as e:\n print(f\"Error saving HTML file: {e}\")\n return False\n\n\ndef add_toc(temp_dir):\n \"\"\"Add TOC to book.html\"\"\"\n print(\"=== Adding Table of Contents ===\")\n\n book_file = os.path.join(temp_dir, 'book.html')\n if not os.path.exists(book_file):\n print(\"Warning: book.html not found, skipping TOC\")\n return False\n\n if BS4_AVAILABLE:\n return insert_toc_with_bs4(book_file)\n else:\n return insert_toc_with_regex(book_file)\n\n\n# =============================================================================\n# Step 7: Generate DOCX/EPUB/PDF with error transparency\n# =============================================================================\n\ndef generate_format(html_file, temp_dir, output_ext, lang_attr, cover=None):\n \"\"\"Generate a specific format using calibre_html_publish.py\"\"\"\n output_file = os.path.join(temp_dir, f\"book{output_ext}\")\n cover = cover if output_ext == '.epub' else None\n if cover and not os.path.isfile(cover):\n print(f\"Cover image not found: {cover}\")\n return None\n\n if os.path.exists(output_file):\n output_mtime = os.path.getmtime(output_file)\n\n # Check if source HTML is newer\n html_newer = os.path.getmtime(html_file) > output_mtime\n\n # Check if any image asset is newer (Calibre embeds these)\n images_newer = False\n images_dir = os.path.join(temp_dir, 'images')\n if os.path.isdir(images_dir):\n for img in os.listdir(images_dir):\n img_path = os.path.join(images_dir, img)\n if os.path.isfile(img_path) and os.path.getmtime(img_path) > output_mtime:\n images_newer = True\n break\n\n cover_newer = bool(cover and os.path.getmtime(cover) > output_mtime)\n\n if not html_newer and not images_newer and not cover_newer:\n file_size = os.path.getsize(output_file)\n print(f\"Skipping {output_ext} - already exists and up to date ({file_size:,} bytes)\")\n return output_file\n else:\n reasons = []\n if html_newer:\n reasons.append(\"source HTML changed\")\n if images_newer:\n reasons.append(\"image assets changed\")\n if cover_newer:\n reasons.append(\"cover image changed\")\n print(f\"Rebuilding {output_ext} - {', '.join(reasons)}\")\n\n publish_script = os.path.join(SCRIPT_DIR, \"calibre_html_publish.py\")\n if not os.path.exists(publish_script):\n print(f\"calibre_html_publish.py not found at: {publish_script}\")\n return None\n\n try:\n cmd = [\"python3\", publish_script, html_file, \"-o\", output_file, \"--lang\", lang_attr]\n if cover:\n cmd.extend([\"--cover\", cover])\n result = subprocess.run(cmd, check=True, capture_output=True, text=True)\n\n if os.path.exists(output_file):\n file_size = os.path.getsize(output_file)\n return output_file\n else:\n print(f\"Failed to generate {output_ext}\")\n if result.stdout:\n print(f\" stdout: {result.stdout[-500:]}\")\n return None\n except subprocess.CalledProcessError as e:\n print(f\"Failed to generate {output_ext}\")\n if e.stdout:\n print(f\" stdout: {e.stdout[-500:]}\")\n if e.stderr:\n print(f\" stderr: {e.stderr[-500:]}\")\n return None\n except Exception as e:\n print(f\"Error generating {output_ext}: {e}\")\n return None\n\n\ndef generate_formats(temp_dir, lang_attr, cover=None):\n \"\"\"Generate DOCX, EPUB, and PDF with result summary\"\"\"\n print(\"=== Generating output formats ===\")\n\n html_file = os.path.join(temp_dir, \"book_doc.html\")\n if not os.path.exists(html_file):\n html_files = glob.glob(os.path.join(temp_dir, \"*.html\"))\n if html_files:\n html_file = max(html_files, key=os.path.getmtime)\n else:\n print(\"No HTML files found for format generation\")\n return\n\n results = {}\n for ext in ['.docx', '.epub', '.pdf']:\n result = generate_format(html_file, temp_dir, ext, lang_attr, cover=cover)\n if result:\n file_size = os.path.getsize(result)\n results[ext] = ('OK', f\"{file_size:,} bytes\")\n else:\n results[ext] = ('FAILED', '')\n\n # Print summary table\n print(\"\\nFormat results:\")\n has_failures = False\n for ext, (status, detail) in results.items():\n if status == 'OK':\n print(f\" {ext}: {status} ({detail})\")\n else:\n print(f\" {ext}: {status}\")\n has_failures = True\n\n return not has_failures\n\n\ndef _validate_export_name(name):\n \"\"\"Validate an export filename stem. Keep aliases inside temp_dir.\"\"\"\n if not name or not name.strip():\n raise ValueError(\"--export-name must not be empty\")\n if '\\x00' in name or '/' in name or '\\\\' in name:\n raise ValueError(\"--export-name must be a filename stem, not a path\")\n stem = name.strip()\n # This stem maps the web HTML alias onto the canonical ebook HTML.\n # Check case-insensitively so the same names are safe on macOS/Windows.\n if stem.casefold() == \"book_doc\":\n raise ValueError(\"--export-name would overwrite reserved output book_doc.html; choose another name\")\n return stem\n\n\ndef export_named_aliases(temp_dir, export_name):\n \"\"\"Copy canonical outputs to optional user-facing filenames.\n\n Canonical artifacts remain untouched. The alias names use export_name as a\n filename stem, with book_doc.html receiving a _doc suffix to avoid colliding\n with the web HTML alias.\n \"\"\"\n stem = _validate_export_name(export_name)\n mappings = {\n \"book.html\": f\"{stem}.html\",\n \"book_doc.html\": f\"{stem}_doc.html\",\n \"book.docx\": f\"{stem}.docx\",\n \"book.epub\": f\"{stem}.epub\",\n \"book.pdf\": f\"{stem}.pdf\",\n }\n copied = []\n for src_name, dst_name in mappings.items():\n src = os.path.join(temp_dir, src_name)\n if not os.path.exists(src):\n continue\n dst = os.path.join(temp_dir, dst_name)\n if os.path.exists(dst) and os.path.samefile(src, dst):\n continue\n shutil.copy2(src, dst)\n copied.append(dst_name)\n return copied\n\n\n# =============================================================================\n# Main\n# =============================================================================\n\ndef main():\n parser = argparse.ArgumentParser(description='Merge translated pages and build final outputs')\n parser.add_argument('--temp-dir', required=True, help='Temp directory path')\n parser.add_argument('--title', default=None, help='Translated book title (override config)')\n parser.add_argument('--author', default=None, help='Author name (override config)')\n parser.add_argument('--lang', default=None, help='Output language code (override config)')\n parser.add_argument('--cover', default=None, help='Cover image path for EPUB output')\n parser.add_argument('--export-name', default=None, help='Optional filename stem for exported alias copies')\n parser.add_argument('--cleanup', action='store_true', help='Remove intermediate artifacts after successful build')\n\n args = parser.parse_args()\n temp_dir = args.temp_dir\n\n if not os.path.isdir(temp_dir):\n print(f\"Error: Temp directory not found: {temp_dir}\")\n sys.exit(1)\n\n cover = args.cover\n if cover:\n if not os.path.isfile(cover):\n print(f\"Error: Cover image not found: {cover}\")\n sys.exit(1)\n cover = os.path.abspath(cover)\n\n export_name = None\n if args.export_name:\n try:\n export_name = _validate_export_name(args.export_name)\n except ValueError as e:\n print(f\"Error: {e}\")\n sys.exit(1)\n\n # Load config as base, CLI args override\n config = load_config(temp_dir)\n\n lang_code = args.lang or config.get('output_lang', 'zh')\n lang_cfg = get_lang_config(lang_code)\n\n title = args.title or config.get('original_title', 'Translated Book')\n author = args.author or config.get('creator', 'Unknown Author')\n\n print(f\"=== Merge and Build ===\")\n print(f\"Temp directory: {temp_dir}\")\n print(f\"Title: {title}\")\n print(f\"Author: {author}\")\n print(f\"Language: {lang_code} (attr: {lang_cfg['lang_attr']})\")\n\n # Step 4: Merge\n if not merge_markdown_files(temp_dir):\n sys.exit(1)\n\n # Step 5: Convert to HTML\n if not convert_md_to_html(temp_dir, title, lang_cfg, author):\n sys.exit(1)\n\n # Step 6: Add TOC\n add_toc(temp_dir)\n\n # Step 7: Generate formats\n all_formats_ok = generate_formats(temp_dir, lang_cfg['lang_attr'], cover=cover)\n\n if export_name:\n if all_formats_ok:\n aliases = export_named_aliases(temp_dir, export_name)\n if aliases:\n print(\"\\nExport aliases:\")\n for name in aliases:\n print(f\" {name}\")\n else:\n print(\"\\nSkipping export aliases — some formats failed.\")\n\n print(\"\\n=== Build Complete ===\")\n print(f\"All outputs saved to: {temp_dir}\")\n\n # List generated files\n for ext in ['book.html', 'book_doc.html', 'book.docx', 'book.epub', 'book.pdf']:\n filepath = os.path.join(temp_dir, ext)\n if os.path.exists(filepath):\n size = os.path.getsize(filepath)\n print(f\" {ext}: {size:,} bytes\")\n\n # Cleanup intermediate artifacts if requested (skip if any format failed)\n if args.cleanup:\n if all_formats_ok:\n cleanup_intermediate_files(temp_dir)\n else:\n print(\"\\nSkipping cleanup — some formats failed. Intermediate files kept for diagnosis/retry.\")\n\n\ndef cleanup_intermediate_files(temp_dir):\n \"\"\"Remove intermediate artifacts, keeping only final outputs.\"\"\"\n print(\"\\n=== Cleaning up intermediate files ===\")\n\n removed = []\n\n # Remove chunk*.md and output_chunk*.md\n for pattern in ['chunk*.md', 'output_chunk*.md']:\n for filepath in glob.glob(os.path.join(temp_dir, pattern)):\n os.remove(filepath)\n removed.append(os.path.basename(filepath))\n\n # Remove specific intermediate files\n for name in ['input.html', 'input.md', 'output.html']:\n filepath = os.path.join(temp_dir, name)\n if os.path.exists(filepath):\n os.remove(filepath)\n removed.append(name)\n\n if removed:\n print(f\"Removed {len(removed)} intermediate file(s):\")\n # Summarize chunk files instead of listing each one\n chunk_files = [f for f in removed if 'chunk' in f]\n other_files = [f for f in removed if 'chunk' not in f]\n if chunk_files:\n print(f\" {len(chunk_files)} chunk files (chunk*.md, output_chunk*.md)\")\n for f in other_files:\n print(f\" {f}\")\n else:\n print(\"No intermediate files to remove.\")\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":45226,"content_sha256":"02fe5ba1a8be6fba9282cfe06a21ad07c89fc19ec1094f164c97c3920e68ee79"},{"filename":"scripts/merge_meta.py","content":"#!/usr/bin/env python3\n\"\"\"\nmerge_meta.py - Merge sub-agent observation files into the canonical glossary.\n\nThree subcommands:\n\n- `prepare-merge \u003ctemp_dir>` — scan all `output_chunk*.meta.json` files in\n the temp dir, filter to those whose content hash isn't already in\n `glossary[\"applied_meta_hashes\"]`, classify findings against current\n glossary state, and emit a JSON proposal to stdout. Malformed meta files\n are quarantined (warn + skip + count separately) — they don't crash the\n step.\n\n- `apply-merge \u003ctemp_dir>` — read a decisions JSON from stdin, conservatively\n apply the auto-applies and resolved decisions to the glossary, and write\n meta content hashes into `applied_meta_hashes` for every chunk in\n `consumed_chunk_ids`. Atomic save.\n\n- `status \u003ctemp_dir>` — read-only observability snapshot for the final\n verification report.\n\nApproach A: scripts NEVER invoke an LLM. All semantic decisions (alias yes/no,\nconflict resolution, conflicting-proposal pick) are made by the main agent in\nSKILL.md Step 4.5; this script just shapes data and applies pre-decided\nchoices.\n\"\"\"\n\nimport argparse\nimport json\nimport os\nimport re\nimport sys\nfrom pathlib import Path\n\nimport glossary as glossary_mod\nimport meta as meta_mod\n\n\nMETA_GLOB = 'output_chunk*.meta.json'\nTRANSLATED_GLOB = 'output_chunk*.md'\nEVIDENCE_REFS_CAP = 5\n\n# Decision kinds and their valid choice sets.\nVALID_CHOICES_BY_KIND = {\n 'alias': frozenset({'yes_alias', 'no_separate_entity', 'skip'}),\n 'conflict': frozenset({'keep_current', 'accept_proposed', 'record_in_notes'}),\n # alias_or_new_entity, conflicting_new_entity_proposals,\n # new_entity_existing_alias, and existing_entity_conflict have dynamic\n # use_*_N choices generated from variant counts; validated at apply time\n # against the decision item's options array.\n}\n\nVALID_GENDER_VALUES = ('male', 'female', 'nonbinary', 'unknown')\n\n\ndef _glossary_path(temp_dir):\n return os.path.join(temp_dir, 'glossary.json')\n\n\ndef _meta_path(temp_dir, chunk_id):\n return os.path.join(temp_dir, f'output_{chunk_id}.meta.json')\n\n\ndef _try_load_meta(path):\n \"\"\"Wrap meta.load_meta for graceful degradation. Returns (data, error_msg).\n\n Honors the \"meta is non-blocking\" promise: a malformed meta file is\n skipped, not raised, so a single bad sub-agent doesn't crash the merge.\n \"\"\"\n try:\n return meta_mod.load_meta(path), None\n except (ValueError, OSError) as e:\n return None, str(e)\n\n\ndef _confidence_for_evidence_count(n):\n if n >= 3:\n return 'high'\n if n == 2:\n return 'medium'\n return 'low'\n\n\n_CONFIDENCE_RANK = {'low': 0, 'medium': 1, 'high': 2}\n\n\ndef _promote_confidence(current, candidate):\n \"\"\"Return the higher-ranked confidence. Never downgrade.\"\"\"\n if _CONFIDENCE_RANK.get(candidate, -1) > _CONFIDENCE_RANK.get(current, -1):\n return candidate\n return current\n\n\ndef _append_evidence_ref(term, chunk_id):\n \"\"\"Append chunk_id to term's evidence_refs (FIFO cap at 5, dedup), then\n promote confidence based on the new length. Returns True if anything\n changed.\"\"\"\n refs = term.get('evidence_refs', [])\n if chunk_id in refs:\n return False\n refs.append(chunk_id)\n if len(refs) > EVIDENCE_REFS_CAP:\n refs = refs[-EVIDENCE_REFS_CAP:]\n term['evidence_refs'] = refs\n term['confidence'] = _promote_confidence(\n term.get('confidence', 'low'),\n _confidence_for_evidence_count(len(refs)),\n )\n return True\n\n\ndef _append_note(term, line):\n \"\"\"Append a line to a term's notes, keeping it readable.\"\"\"\n existing = term.get('notes', '') or ''\n term['notes'] = (existing + '\\n' + line).lstrip('\\n')\n\n\ndef _build_surface_index(glossary):\n \"\"\"Map every surface form (source or alias) → (term_id, role).\"\"\"\n idx = {}\n for t in glossary['terms']:\n idx[t['source']] = (t['id'], 'source')\n for a in t.get('aliases', []):\n idx[a] = (t['id'], 'alias')\n return idx\n\n\ndef _find_term_by_id(glossary, entity_id):\n for t in glossary['terms']:\n if t['id'] == entity_id:\n return t\n return None\n\n\ndef _find_term_by_surface(glossary, surface):\n \"\"\"Return the term whose source OR any alias equals `surface`.\"\"\"\n for t in glossary['terms']:\n if t['source'] == surface or surface in t.get('aliases', []):\n return t\n return None\n\n\ndef cmd_prepare_merge(temp_dir):\n \"\"\"Scan unmerged metas and propose a merge plan. Output JSON on stdout.\"\"\"\n glossary_path = _glossary_path(temp_dir)\n glossary = glossary_mod.load_glossary(glossary_path)\n applied = glossary.get('applied_meta_hashes', {})\n\n meta_paths = sorted(Path(temp_dir).glob(META_GLOB))\n\n consumed_chunk_ids = []\n malformed_chunk_ids = []\n # Loaded metas keyed by chunk_id, only including non-malformed unmerged ones.\n loaded = {}\n\n for path in meta_paths:\n try:\n chunk_id = meta_mod.chunk_id_from_meta_path(str(path))\n except ValueError as e:\n sys.stderr.write(f\"warn: {e}\\n\")\n continue\n\n data, err = _try_load_meta(str(path))\n if err is not None:\n sys.stderr.write(\n f\"warn: malformed meta for {chunk_id} at {path} — quarantined ({err})\\n\"\n )\n malformed_chunk_ids.append(chunk_id)\n continue\n\n content_hash = meta_mod.meta_content_hash(data)\n if applied.get(chunk_id) == content_hash:\n continue # already applied, skip silently\n\n loaded[chunk_id] = data\n consumed_chunk_ids.append(chunk_id)\n\n # Group new_entities across all loaded metas by source string.\n grouped_new_entities = {}\n for chunk_id, data in loaded.items():\n for entity in data.get('new_entities', []):\n src = entity['source']\n grouped_new_entities.setdefault(src, []).append({\n 'chunk_id': chunk_id,\n 'target_proposal': entity['target_proposal'],\n 'category': entity.get('category', ''),\n 'evidence': entity['evidence'],\n })\n\n surface_idx = _build_surface_index(glossary)\n\n auto_apply = []\n decisions_needed = []\n next_decision_id = 1\n\n def _new_decision_id():\n nonlocal next_decision_id\n did = f'd{next_decision_id}'\n next_decision_id += 1\n return did\n\n # Helper: build the standalone_variants array for a variant pulled out\n # of grouped_new_entities. Each (target, category) pair gets its own\n # variant entry with its evidence_chunks, parallel to conflicting_new_entity_proposals.\n def _standalone_variants_for(proposals):\n grouped_by_pair = {}\n for p in proposals:\n grouped_by_pair.setdefault((p['target_proposal'], p['category']), []).append(p)\n out = []\n for (target, category), ps in grouped_by_pair.items():\n out.append({\n 'target_proposal': target,\n 'category': category,\n 'evidence': ps[0]['evidence'],\n 'evidence_chunks': sorted({p['chunk_id'] for p in ps}),\n })\n return out\n\n # Group alias_hypotheses by variant across all loaded metas. Each variant\n # may have multiple alias candidates (e.g. chunk0002 says Taig→Tai while\n # chunk0003 says Taig→Tao); they're all options the orchestrator must\n # choose between. Multiple chunks proposing the same (variant, candidate)\n # pair contribute their evidence_chunks to a single candidate entry.\n grouped_alias_hyps = {} # variant -> list of (chunk_id, ah)\n for chunk_id, data in loaded.items():\n for ah in data.get('alias_hypotheses', []):\n grouped_alias_hyps.setdefault(ah['variant'], []).append((chunk_id, ah))\n\n def _alias_candidates_for(hyps):\n \"\"\"Aggregate (chunk_id, ah) tuples into per-candidate entries with\n combined evidence_chunks. Preserves the sub-agent's literal candidate\n string — apply-merge re-resolves at dispatch time.\"\"\"\n by_candidate = {}\n for chunk_id, ah in hyps:\n by_candidate.setdefault(ah['may_be_alias_of_source'], []).append((chunk_id, ah))\n out = []\n for candidate, items in by_candidate.items():\n out.append({\n 'candidate_source': candidate,\n 'evidence': items[0][1]['evidence'],\n 'evidence_chunks': sorted({cid for cid, _ in items}),\n })\n return out\n\n # Pass 1: variants that collide with same-batch standalone proposals.\n # Emit alias_or_new_entity exposing EVERY alias candidate AND every\n # standalone variant. Pop the variant from grouped_new_entities so it\n # doesn't also get auto_apply'd / decisioned below.\n consumed_alias_variants = set() # variants already wrapped into a unified decision\n for variant in list(grouped_alias_hyps.keys()):\n if variant not in grouped_new_entities:\n continue\n if variant in surface_idx:\n # Variant is already a glossary surface; the\n # new_entity_existing_alias / conflict path owns it.\n continue\n alias_candidates = _alias_candidates_for(grouped_alias_hyps[variant])\n proposals = grouped_new_entities.pop(variant)\n standalone_variants = _standalone_variants_for(proposals)\n options = (\n [f'use_alias_{i}' for i in range(len(alias_candidates))]\n + [f'use_standalone_{i}' for i in range(len(standalone_variants))]\n + ['skip']\n )\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'alias_or_new_entity',\n 'variant': variant,\n 'alias_candidates': alias_candidates,\n 'standalone_variants': standalone_variants,\n 'options': options,\n })\n consumed_alias_variants.add(variant)\n\n # Pass 2: variants with 2+ alias candidates but NO standalone collision.\n # Without a unified decision the orchestrator could naively accept multiple\n # `yes_alias` decisions for the same variant — surface uniqueness violation.\n for variant in list(grouped_alias_hyps.keys()):\n if variant in consumed_alias_variants:\n continue\n if variant in surface_idx:\n continue\n alias_candidates = _alias_candidates_for(grouped_alias_hyps[variant])\n if len(alias_candidates) \u003c 2:\n continue\n options = [f'use_alias_{i}' for i in range(len(alias_candidates))] + ['skip']\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'alias_or_new_entity',\n 'variant': variant,\n 'alias_candidates': alias_candidates,\n 'standalone_variants': [],\n 'options': options,\n })\n consumed_alias_variants.add(variant)\n\n for src, proposals in grouped_new_entities.items():\n target_cat_pairs = {(p['target_proposal'], p['category']) for p in proposals}\n if src in surface_idx:\n owner_id, owner_role = surface_idx[src]\n owner_term = _find_term_by_id(glossary, owner_id)\n if owner_role == 'source':\n # src is already canonical. For each distinct (target, category)\n # proposal across the batch that DIFFERS from the canonical\n # (target, category), expose it as a competing variant. If\n # every proposal matches canonical, silent no-op.\n canonical_pair = (\n owner_term['target'],\n owner_term.get('category', ''),\n )\n proposed_variants = _standalone_variants_for(proposals)\n differing = [\n v for v in proposed_variants\n if (v['target_proposal'], v['category']) != canonical_pair\n ]\n if differing:\n options = (\n ['keep_current']\n + [f'use_variant_{i}' for i in range(len(differing))]\n + ['record_in_notes']\n )\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'existing_entity_conflict',\n 'entity_source': src,\n 'current_target': owner_term['target'],\n 'current_category': owner_term.get('category', ''),\n 'proposed_variants': differing,\n 'options': options,\n })\n # else: every proposal matched canonical — no decision needed.\n else: # role == 'alias'\n # src is currently another term's alias; sub-agents propose to\n # promote it. Expose every distinct (target, category) variant\n # so the orchestrator can pick the right promoted form.\n promoted_variants = _standalone_variants_for(proposals)\n options = (\n [f'use_variant_{i}' for i in range(len(promoted_variants))]\n + ['keep_as_alias', 'skip']\n )\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'new_entity_existing_alias',\n 'proposed_source': src,\n 'currently_alias_of': owner_id,\n 'promoted_variants': promoted_variants,\n 'options': options,\n })\n elif len(target_cat_pairs) == 1:\n # All proposals agree → auto_apply with combined evidence.\n target, category = next(iter(target_cat_pairs))\n chunk_ids = sorted({p['chunk_id'] for p in proposals})\n auto_apply.append({\n 'action': 'add_entity',\n 'entity': {\n 'source': src,\n 'target_proposal': target,\n 'category': category,\n 'evidence': proposals[0]['evidence'],\n },\n 'evidence_chunks': chunk_ids,\n })\n else:\n # Conflicting proposals across chunks — main agent must pick.\n variants = _standalone_variants_for(proposals)\n options = [f'use_variant_{i}' for i in range(len(variants))] + ['skip']\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'conflicting_new_entity_proposals',\n 'source': src,\n 'variants': variants,\n 'options': options,\n })\n\n # Compute the set of every surface that's already a glossary source OR\n # could become a standalone source via this batch's auto_apply / decisions.\n # The variant collision check uses this — we don't emit a regular alias\n # decision for a variant that another decision kind already owns.\n potential_standalone_sources = set(surface_idx.keys()) # everything currently in glossary\n for entry in auto_apply:\n potential_standalone_sources.add(entry['entity']['source'])\n for d in decisions_needed:\n if d['kind'] == 'conflicting_new_entity_proposals':\n potential_standalone_sources.add(d['source'])\n elif d['kind'] == 'new_entity_existing_alias':\n potential_standalone_sources.add(d['proposed_source'])\n elif d['kind'] == 'alias_or_new_entity':\n potential_standalone_sources.add(d['variant'])\n\n # Pass 3: simple alias hypotheses (single candidate, no standalone collision).\n # Emit a decision if the candidate is — or could become — any glossary\n # surface. Variant must NOT also be a potential standalone source; those\n # collisions are owned by alias_or_new_entity / new_entity_existing_alias.\n #\n # Alias chains are handled by iterating until fixed point: a hypothesis\n # `Taighi → Taig` is moot when first inspected if Taig isn't anywhere yet,\n # but it becomes resolvable after a sibling hypothesis `Taig → Tai` is\n # emitted — Taig will be a glossary alias by apply time. We loop until no\n # new decisions get emitted in a pass.\n #\n # candidate_source is preserved verbatim — apply-merge re-resolves at\n # dispatch time, after all entity-creating decisions have run AND after\n # earlier alias attachments have made new surfaces resolvable.\n pending_alias_hyps = []\n for variant, hyps in grouped_alias_hyps.items():\n if variant in consumed_alias_variants:\n continue # owned by alias_or_new_entity above\n # By construction (pass 2), any variant reaching here has exactly one\n # distinct alias candidate. Multiple chunks proposing the same\n # (variant, candidate) collapse to a single decision; we keep the\n # first hypothesis as the representative (its evidence quote shows\n # in the prompt; the other chunks' hashes are still recorded via\n # consumed_chunk_ids when the decision is applied).\n first_chunk_id, first_ah = hyps[0]\n pending_alias_hyps.append((first_chunk_id, first_ah))\n\n # potential_surfaces = anything that might be a resolvable surface at apply\n # time. Starts with current glossary surfaces + every potential standalone\n # source. Grows as we emit alias decisions (each new alias variant becomes\n # a resolvable surface once attached to its candidate's term).\n potential_surfaces = set(surface_idx.keys()) | potential_standalone_sources\n\n while pending_alias_hyps:\n next_pending = []\n progressed = False\n for chunk_id, ah in pending_alias_hyps:\n variant = ah['variant']\n candidate_source = ah['may_be_alias_of_source']\n if candidate_source not in potential_surfaces:\n next_pending.append((chunk_id, ah))\n continue\n if variant in potential_standalone_sources:\n # Variant is owned by a standalone-creating decision elsewhere;\n # silently skip (was the moot rule).\n continue\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'alias',\n 'variant': variant,\n 'candidate_source': candidate_source,\n 'evidence': ah['evidence'],\n 'options': ['yes_alias', 'no_separate_entity', 'skip'],\n })\n potential_surfaces.add(variant) # downstream chains can now resolve through it\n progressed = True\n if not progressed:\n break # remaining hypotheses are truly moot\n pending_alias_hyps = next_pending\n\n # Sub-agent-flagged conflicts.\n for chunk_id, data in loaded.items():\n for c in data.get('conflicts', []):\n entity_term = _find_term_by_surface(glossary, c['entity_source'])\n if entity_term is None:\n continue # nothing to conflict with\n field = c['field']\n current_val = entity_term.get(field, '')\n decisions_needed.append({\n 'id': _new_decision_id(),\n 'kind': 'conflict',\n 'entity_source': c['entity_source'],\n 'field': field,\n 'current': current_val,\n 'proposed': c['observed_better'],\n 'evidence': c['evidence'],\n 'options': ['keep_current', 'accept_proposed', 'record_in_notes'],\n })\n\n output = {\n 'auto_apply': auto_apply,\n 'decisions_needed': decisions_needed,\n 'consumed_chunk_ids': sorted(consumed_chunk_ids),\n 'malformed_meta_chunk_ids': sorted(malformed_chunk_ids),\n }\n print(json.dumps(output, ensure_ascii=False, indent=2))\n\n\ndef cmd_apply_merge(temp_dir):\n \"\"\"Read decisions JSON from stdin and apply to the glossary.\"\"\"\n try:\n decisions_doc = json.load(sys.stdin)\n except json.JSONDecodeError as e:\n sys.stderr.write(f\"error: decisions JSON on stdin is not valid: {e}\\n\")\n sys.exit(2)\n\n if not isinstance(decisions_doc, dict):\n sys.stderr.write(\"error: decisions JSON must be an object\\n\")\n sys.exit(2)\n\n auto_apply = decisions_doc.get('auto_apply', [])\n decisions = decisions_doc.get('decisions', [])\n consumed_chunk_ids = decisions_doc.get('consumed_chunk_ids', [])\n\n if not isinstance(auto_apply, list):\n sys.stderr.write(\"error: 'auto_apply' must be a list\\n\")\n sys.exit(2)\n if not isinstance(decisions, list):\n sys.stderr.write(\"error: 'decisions' must be a list\\n\")\n sys.exit(2)\n if not isinstance(consumed_chunk_ids, list):\n sys.stderr.write(\"error: 'consumed_chunk_ids' must be a list\\n\")\n sys.exit(2)\n\n glossary_path = _glossary_path(temp_dir)\n glossary = glossary_mod.load_glossary(glossary_path)\n\n auto_applied = 0\n decisions_resolved = 0\n errors = []\n\n # Phase 1: process every consumed meta — used_term_sources confidence escalation\n # AND content hash recording. This runs even with empty auto_apply / decisions.\n consumed_metas = {} # chunk_id -> meta data\n for chunk_id in consumed_chunk_ids:\n meta_path = _meta_path(temp_dir, chunk_id)\n if not os.path.exists(meta_path):\n errors.append(f\"no meta file for chunk_id={chunk_id} at {meta_path}\")\n continue\n # Defensive: re-derive chunk_id from filename and check.\n derived = meta_mod.chunk_id_from_meta_path(meta_path)\n if derived != chunk_id:\n errors.append(\n f\"chunk_id mismatch: consumed_chunk_ids has {chunk_id!r}, \"\n f\"filename derives {derived!r}\"\n )\n continue\n data, err = _try_load_meta(meta_path)\n if err is not None:\n errors.append(f\"meta for chunk_id={chunk_id} is malformed: {err}\")\n continue\n consumed_metas[chunk_id] = data\n\n if errors:\n sys.stderr.write(\"error: apply-merge aborting before any state mutation:\\n\")\n for e in errors:\n sys.stderr.write(f\" - {e}\\n\")\n sys.exit(2)\n\n # Pre-validate every decision payload BEFORE we mutate. If any decision is\n # malformed (wrong kind, invalid choice for kind, missing fields) we abort\n # the entire transaction without writing any hashes — otherwise on retry\n # prepare-merge would skip the affected metas and the unresolved decisions\n # would be lost forever.\n pre_errors = []\n for d in decisions:\n d_id = d.get('id')\n kind = d.get('kind')\n choice = d.get('choice')\n if kind is None or choice is None:\n pre_errors.append(f\"decision {d_id!r} missing 'kind' or 'choice'\")\n continue\n if kind == 'conflicting_new_entity_proposals':\n variants = d.get('variants') or []\n valid = {f'use_variant_{i}' for i in range(len(variants))} | {'skip'}\n if choice not in valid:\n pre_errors.append(\n f\"decision {d_id!r} (kind={kind}): invalid choice {choice!r}, \"\n f\"must be one of {sorted(valid)}\"\n )\n continue\n if kind == 'alias_or_new_entity':\n alias_candidates = d.get('alias_candidates') or []\n standalone_variants = d.get('standalone_variants') or []\n valid = (\n {'skip'}\n | {f'use_alias_{i}' for i in range(len(alias_candidates))}\n | {f'use_standalone_{i}' for i in range(len(standalone_variants))}\n )\n if choice not in valid:\n pre_errors.append(\n f\"decision {d_id!r} (kind={kind}): invalid choice {choice!r}, \"\n f\"must be one of {sorted(valid)}\"\n )\n continue\n if kind == 'new_entity_existing_alias':\n promoted_variants = d.get('promoted_variants') or []\n valid = (\n {'keep_as_alias', 'skip'}\n | {f'use_variant_{i}' for i in range(len(promoted_variants))}\n )\n if choice not in valid:\n pre_errors.append(\n f\"decision {d_id!r} (kind={kind}): invalid choice {choice!r}, \"\n f\"must be one of {sorted(valid)}\"\n )\n continue\n if kind == 'existing_entity_conflict':\n proposed_variants = d.get('proposed_variants') or []\n valid = (\n {'keep_current', 'record_in_notes'}\n | {f'use_variant_{i}' for i in range(len(proposed_variants))}\n )\n if choice not in valid:\n pre_errors.append(\n f\"decision {d_id!r} (kind={kind}): invalid choice {choice!r}, \"\n f\"must be one of {sorted(valid)}\"\n )\n continue\n valid = VALID_CHOICES_BY_KIND.get(kind)\n if valid is None:\n pre_errors.append(f\"decision {d_id!r}: unknown kind {kind!r}\")\n continue\n if choice not in valid:\n pre_errors.append(\n f\"decision {d_id!r} (kind={kind}): invalid choice {choice!r}, \"\n f\"must be one of {sorted(valid)}\"\n )\n\n if pre_errors:\n sys.stderr.write(\"error: apply-merge aborting before any state mutation:\\n\")\n for e in pre_errors:\n sys.stderr.write(f\" - {e}\\n\")\n sys.exit(2)\n\n # Now safe to mutate. Process used_term_sources first (purely additive).\n for chunk_id, data in consumed_metas.items():\n for src in data.get('used_term_sources', []):\n term = _find_term_by_surface(glossary, src)\n if term is None:\n continue\n _append_evidence_ref(term, chunk_id)\n\n # Process attribute_hypotheses (currently: gender). Roadmap rule:\n # unknown → first explicit evidence sets value + promotes confidence;\n # corroborating evidence keeps value + promotes confidence; conflicting\n # evidence resets to unknown and records both sides in notes.\n for chunk_id, data in consumed_metas.items():\n for ah in data.get('attribute_hypotheses', []):\n entity_source = ah.get('entity_source')\n attribute = ah.get('attribute')\n value = ah.get('value')\n evidence = ah.get('evidence', '')\n term = _find_term_by_surface(glossary, entity_source)\n if term is None:\n continue\n if attribute == 'gender':\n if value not in VALID_GENDER_VALUES:\n # Schema validation accepted any string; we silently no-op\n # on values the glossary doesn't model.\n continue\n current = term.get('gender', 'unknown')\n if current == 'unknown':\n term['gender'] = value\n _append_evidence_ref(term, chunk_id)\n _append_note(\n term,\n f\"[gender] {value} set (chunk {chunk_id}); evidence={evidence!r}\"\n )\n elif current == value:\n # Corroborating — promote entity confidence.\n _append_evidence_ref(term, chunk_id)\n else:\n # Conflict: revert to unknown, record both observations.\n term['gender'] = 'unknown'\n _append_note(\n term,\n f\"[gender] conflict — was {current!r}, observed {value!r} \"\n f\"in chunk {chunk_id}; evidence={evidence!r}; reverted to 'unknown'\"\n )\n # Other attributes: log to notes but don't mutate canonical fields.\n else:\n _append_note(\n term,\n f\"[attr {attribute!r}] observed {value!r} in chunk {chunk_id}; \"\n f\"evidence={evidence!r}\"\n )\n\n # Phase 2: auto_apply new entities.\n for entry in auto_apply:\n if entry.get('action') != 'add_entity':\n errors.append(f\"unknown auto_apply action: {entry.get('action')!r}\")\n continue\n entity = entry['entity']\n evidence_chunks = list(entry.get('evidence_chunks', []))[:EVIDENCE_REFS_CAP]\n new_term = {\n 'id': entity['source'],\n 'source': entity['source'],\n 'target': entity['target_proposal'],\n 'category': entity.get('category', ''),\n 'aliases': [],\n 'gender': 'unknown',\n 'confidence': _confidence_for_evidence_count(len(evidence_chunks)),\n 'frequency': 0,\n 'evidence_refs': evidence_chunks,\n 'notes': '',\n }\n glossary['terms'].append(new_term)\n auto_applied += 1\n\n # Phase 3: process decisions. All choices have been pre-validated above.\n #\n # Dispatch order is INDEPENDENT of input order:\n # Pass A — entity-creating decisions, in input order\n # Pass B — non-alias followers (conflicts + no-op skips), in input order\n # Pass C — alias-attachers (alias.yes_alias, alias_or_new_entity.yes_alias),\n # iterated to fixed point so chains like Taighi → Taig → Tai\n # resolve regardless of which alias decision came first.\n #\n # Any referential failure (entity not in glossary, etc.) is collected; if\n # non-empty after all passes, the entire transaction aborts so the\n # orchestrator can re-attempt.\n\n def _is_creator(d):\n kind = d['kind']\n choice = d['choice']\n if kind == 'new_entity_existing_alias':\n return choice.startswith('use_variant_')\n if kind == 'alias_or_new_entity':\n return choice.startswith('use_standalone_')\n if kind == 'conflicting_new_entity_proposals':\n return choice.startswith('use_variant_')\n return False\n\n def _is_alias_attacher(d):\n kind = d['kind']\n choice = d['choice']\n if kind == 'alias':\n return choice == 'yes_alias'\n if kind == 'alias_or_new_entity':\n return choice.startswith('use_alias_')\n return False\n\n def _dispatch(d):\n \"\"\"Apply one decision. Returns (changed, err_or_None). `changed` is\n True if the dispatch did meaningful work (or was a real no-op like\n keep_current), False only if a referential lookup failed and the\n caller should retry later.\"\"\"\n d_id = d.get('id')\n kind = d['kind']\n choice = d['choice']\n\n if kind == 'alias':\n if choice == 'yes_alias':\n variant = d.get('variant')\n candidate_source = d.get('candidate_source')\n term = _find_term_by_surface(glossary, candidate_source)\n if term is None:\n return False, (\n f\"decision {d_id!r}: candidate source {candidate_source!r} \"\n f\"not in glossary\"\n )\n if variant not in term.get('aliases', []):\n term.setdefault('aliases', []).append(variant)\n return True, None\n\n if kind == 'conflict':\n entity_source = d.get('entity_source')\n field = d.get('field')\n current = d.get('current')\n proposed = d.get('proposed')\n evidence = d.get('evidence', '')\n term = _find_term_by_surface(glossary, entity_source)\n if term is None:\n return False, (\n f\"decision {d_id!r}: entity_source {entity_source!r} not in glossary\"\n )\n if choice == 'record_in_notes':\n _append_note(\n term,\n f\"[conflict] {field}: current={current!r} \"\n f\"observed_better={proposed!r} evidence={evidence!r}\",\n )\n elif choice == 'accept_proposed':\n old_value = term.get(field)\n term[field] = proposed\n _append_note(\n term,\n f\"[updated] {field}: was={old_value!r} now={proposed!r}\",\n )\n # keep_current: no-op\n return True, None\n\n if kind == 'new_entity_existing_alias':\n promoted_variants = d.get('promoted_variants') or []\n if choice in ('keep_as_alias', 'skip'):\n return True, None\n m = re.match(r'^use_variant_(\\d+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, choice)\n idx = int(m.group(1))\n chosen = promoted_variants[idx]\n proposed_source = d.get('proposed_source')\n host_id = d.get('currently_alias_of')\n host = _find_term_by_id(glossary, host_id)\n if host is None:\n return False, (\n f\"decision {d_id!r}: host term id={host_id!r} not in glossary\"\n )\n if proposed_source in host.get('aliases', []):\n host['aliases'] = [a for a in host['aliases'] if a != proposed_source]\n combined_chunks = sorted({\n cid for v in promoted_variants for cid in v.get('evidence_chunks', [])\n })[:EVIDENCE_REFS_CAP]\n glossary['terms'].append({\n 'id': proposed_source,\n 'source': proposed_source,\n 'target': chosen['target_proposal'],\n 'category': chosen.get('category', ''),\n 'aliases': [],\n 'gender': 'unknown',\n 'confidence': _confidence_for_evidence_count(len(combined_chunks)),\n 'frequency': 0,\n 'evidence_refs': combined_chunks,\n 'notes': f'promoted from alias of {host_id!r}',\n })\n return True, None\n\n if kind == 'existing_entity_conflict':\n entity_source = d.get('entity_source')\n term = _find_term_by_surface(glossary, entity_source)\n if term is None:\n return False, (\n f\"decision {d_id!r}: entity_source {entity_source!r} not in glossary\"\n )\n if choice == 'keep_current':\n return True, None\n proposed_variants = d.get('proposed_variants') or []\n if choice == 'record_in_notes':\n for v in proposed_variants:\n _append_note(\n term,\n f\"[conflict] target={v['target_proposal']!r} \"\n f\"category={v['category']!r} evidence={v['evidence']!r}\",\n )\n return True, None\n m = re.match(r'^use_variant_(\\d+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, choice)\n idx = int(m.group(1))\n chosen = proposed_variants[idx]\n old_target = term.get('target')\n old_category = term.get('category', '')\n term['target'] = chosen['target_proposal']\n term['category'] = chosen.get('category', '')\n _append_note(\n term,\n f\"[updated] target {old_target!r} → {chosen['target_proposal']!r}, \"\n f\"category {old_category!r} → {chosen.get('category', '')!r}\",\n )\n return True, None\n\n if kind == 'alias_or_new_entity':\n variant = d.get('variant')\n alias_candidates = d.get('alias_candidates') or []\n standalone_variants = d.get('standalone_variants') or []\n if choice == 'skip':\n return True, None\n m_alias = re.match(r'^use_alias_(\\d+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, choice)\n if m_alias:\n idx = int(m_alias.group(1))\n candidate_source = alias_candidates[idx]['candidate_source']\n term = _find_term_by_surface(glossary, candidate_source)\n if term is None:\n return False, (\n f\"decision {d_id!r}: candidate source {candidate_source!r} \"\n f\"not in glossary\"\n )\n if variant not in term.get('aliases', []):\n term.setdefault('aliases', []).append(variant)\n return True, None\n # use_standalone_N\n m = re.match(r'^use_standalone_(\\d+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, choice)\n idx = int(m.group(1))\n chosen = standalone_variants[idx]\n combined_chunks = sorted({\n cid for v in standalone_variants for cid in v.get('evidence_chunks', [])\n })[:EVIDENCE_REFS_CAP]\n glossary['terms'].append({\n 'id': variant,\n 'source': variant,\n 'target': chosen['target_proposal'],\n 'category': chosen.get('category', ''),\n 'aliases': [],\n 'gender': 'unknown',\n 'confidence': _confidence_for_evidence_count(len(combined_chunks)),\n 'frequency': 0,\n 'evidence_refs': combined_chunks,\n 'notes': '',\n })\n return True, None\n\n if kind == 'conflicting_new_entity_proposals':\n variants = d.get('variants') or []\n if choice == 'skip':\n return True, None\n m = re.match(r'^use_variant_(\\d+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, choice)\n idx = int(m.group(1))\n chosen = variants[idx]\n combined_chunks = sorted({\n cid for v in variants for cid in v.get('evidence_chunks', [])\n })[:EVIDENCE_REFS_CAP]\n glossary['terms'].append({\n 'id': d['source'],\n 'source': d['source'],\n 'target': chosen['target_proposal'],\n 'category': chosen.get('category', ''),\n 'aliases': [],\n 'gender': 'unknown',\n 'confidence': _confidence_for_evidence_count(len(combined_chunks)),\n 'frequency': 0,\n 'evidence_refs': combined_chunks,\n 'notes': '',\n })\n return True, None\n\n # Should never reach here — pre-validation rejects unknown kinds.\n return True, None\n\n creators = [d for d in decisions if _is_creator(d)]\n attachers = [d for d in decisions if _is_alias_attacher(d)]\n others = [d for d in decisions if not _is_creator(d) and not _is_alias_attacher(d)]\n\n # Pass A: entity-creators (in input order).\n for d in creators:\n changed, err = _dispatch(d)\n if err is not None:\n errors.append(err)\n else:\n decisions_resolved += 1\n\n # Pass B: non-alias followers (in input order).\n for d in others:\n changed, err = _dispatch(d)\n if err is not None:\n errors.append(err)\n else:\n decisions_resolved += 1\n\n # Pass C: alias-attachers iterated to fixed point. A chain like\n # Taighi → Taig → Tai: when Taig is itself a yes_alias decision, the first\n # iteration may not be able to resolve Taighi → Taig because Taig isn't a\n # surface yet. The next iteration succeeds after Taig → Tai attaches.\n pending_attachers = list(attachers)\n while pending_attachers:\n next_pending = []\n progressed = False\n for d in pending_attachers:\n changed, err = _dispatch(d)\n if err is None:\n decisions_resolved += 1\n progressed = True\n else:\n next_pending.append((d, err))\n if not progressed:\n for _, err in next_pending:\n errors.append(err)\n break\n pending_attachers = [d for d, _ in next_pending]\n\n if errors:\n # Don't write the glossary, don't record any hashes. Orchestrator\n # retries with fixed payload; prepare-merge will surface the same\n # decisions again because no hashes were recorded.\n sys.stderr.write(\"error: apply-merge failed during decision dispatch; \"\n \"aborting transaction (no glossary or hash mutations persisted):\\n\")\n for e in errors:\n sys.stderr.write(f\" - {e}\\n\")\n sys.exit(2)\n\n # Phase 4: record content hashes for all consumed metas. Reached only\n # when every decision succeeded.\n glossary.setdefault('applied_meta_hashes', {})\n for chunk_id, data in consumed_metas.items():\n glossary['applied_meta_hashes'][chunk_id] = meta_mod.meta_content_hash(data)\n\n # Validate before save (save_glossary will re-validate too — this surfaces\n # surface-uniqueness violations from a bad alias decision before we touch disk).\n try:\n glossary_mod.save_glossary(glossary_path, glossary)\n except ValueError as e:\n sys.stderr.write(f\"error: post-merge glossary failed validation; aborting: {e}\\n\")\n sys.exit(2)\n\n summary = {\n 'auto_applied': auto_applied,\n 'decisions_resolved': decisions_resolved,\n 'consumed_chunks': len(consumed_metas),\n 'errors': errors,\n }\n print(json.dumps(summary, ensure_ascii=False, indent=2))\n\n\ndef cmd_status(temp_dir):\n \"\"\"Read-only summary for the final verification report.\"\"\"\n glossary_path = _glossary_path(temp_dir)\n if os.path.exists(glossary_path):\n glossary = glossary_mod.load_glossary(glossary_path)\n applied = glossary.get('applied_meta_hashes', {})\n else:\n applied = {}\n\n translated_paths = sorted(Path(temp_dir).glob(TRANSLATED_GLOB))\n # Strip the .meta.json variants that the glob also catches if any.\n translated_paths = [p for p in translated_paths if not p.name.endswith('.meta.json')]\n translated_chunk_ids = set()\n for p in translated_paths:\n m = re.match(r'^output_(chunk\\d+)\\.md

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, p.name)\n if m:\n translated_chunk_ids.add(m.group(1))\n\n meta_paths = sorted(Path(temp_dir).glob(META_GLOB))\n found_chunk_ids = set()\n consumed_chunk_ids = set()\n malformed_chunk_ids = []\n unmerged_chunk_ids = []\n for path in meta_paths:\n try:\n chunk_id = meta_mod.chunk_id_from_meta_path(str(path))\n except ValueError:\n continue\n found_chunk_ids.add(chunk_id)\n data, err = _try_load_meta(str(path))\n if err is not None:\n malformed_chunk_ids.append(chunk_id)\n continue\n if applied.get(chunk_id) == meta_mod.meta_content_hash(data):\n consumed_chunk_ids.add(chunk_id)\n else:\n unmerged_chunk_ids.append(chunk_id)\n\n missing_meta = sorted(translated_chunk_ids - found_chunk_ids)\n\n out = {\n 'translated_chunks': len(translated_chunk_ids),\n 'meta_files_found': len(found_chunk_ids),\n 'meta_files_consumed': len(consumed_chunk_ids),\n 'unmerged_meta_files': len(unmerged_chunk_ids),\n 'malformed_meta_files': len(malformed_chunk_ids),\n 'missing_meta_chunk_ids': missing_meta[:10],\n 'malformed_meta_chunk_ids': sorted(malformed_chunk_ids)[:10],\n }\n print(json.dumps(out, ensure_ascii=False, indent=2))\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Merge sub-agent meta files into glossary\")\n sub = parser.add_subparsers(dest='cmd', required=True)\n\n p_prep = sub.add_parser('prepare-merge', help=\"Scan unmerged metas; emit merge proposal\")\n p_prep.add_argument('temp_dir')\n\n p_app = sub.add_parser('apply-merge', help=\"Apply merge decisions from stdin\")\n p_app.add_argument('temp_dir')\n\n p_stat = sub.add_parser('status', help=\"Read-only observability snapshot\")\n p_stat.add_argument('temp_dir')\n\n args = parser.parse_args()\n\n if args.cmd == 'prepare-merge':\n cmd_prepare_merge(args.temp_dir)\n elif args.cmd == 'apply-merge':\n cmd_apply_merge(args.temp_dir)\n elif args.cmd == 'status':\n cmd_status(args.temp_dir)\n\n\nif __name__ == '__main__':\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":43840,"content_sha256":"844e73544d1c4c96c9fd2d6a13010a25665375ceb091324d57b758a3f0f5b376"},{"filename":"scripts/meta.py","content":"#!/usr/bin/env python3\n\"\"\"\nmeta.py - Per-chunk sub-agent observation file.\n\nEach translation sub-agent emits an `output_chunk\u003cNNNN>.meta.json` alongside\nits translated chunk. The main agent reads these after each batch and merges\nthem into the canonical glossary (see merge_meta.py).\n\nThe schema (v1):\n\n {\n \"schema_version\": 1,\n \"new_entities\": [{\"source\": \"...\", \"target_proposal\": \"...\",\n \"category\": \"...\", \"evidence\": \"...\"}],\n \"alias_hypotheses\": [{\"variant\": \"...\", \"may_be_alias_of_source\": \"...\",\n \"evidence\": \"...\"}],\n \"attribute_hypotheses\": [{\"entity_source\": \"...\", \"attribute\": \"...\",\n \"value\": \"...\", \"confidence\": \"...\",\n \"evidence\": \"...\"}],\n \"used_term_sources\": [\"...\"],\n \"conflicts\": [{\"entity_source\": \"...\", \"field\": \"...\",\n \"injected\": \"...\", \"observed_better\": \"...\",\n \"evidence\": \"...\"}]\n }\n\n`chunk_id` is intentionally NOT a field — chunk identity is derived from the\nfilename (`output_chunk\u003cNNNN>.meta.json` → `chunk\u003cNNNN>`). Letting the\nsub-agent fill it would create a hallucination hole.\n\"\"\"\n\nimport hashlib\nimport json\nimport os\nimport re\nimport tempfile\n\n\nMETA_SCHEMA_VERSION = 1\nEVIDENCE_MAX_LEN = 200\n\nVALID_TOP_LEVEL_KEYS = frozenset({\n 'schema_version',\n 'new_entities',\n 'alias_hypotheses',\n 'attribute_hypotheses',\n 'used_term_sources',\n 'conflicts',\n})\n\nVALID_ATTRIBUTE_CONFIDENCES = ('low', 'medium', 'high')\n\n_CHUNK_ID_RE = re.compile(r'^output_(chunk\\d+)\\.meta\\.json

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

)\n\n\ndef chunk_id_from_meta_path(path):\n \"\"\"Extract the chunk identifier from a meta filename.\n\n Filename is the authoritative source of chunk identity — never trust the\n payload to provide it.\n \"\"\"\n basename = os.path.basename(path)\n m = _CHUNK_ID_RE.match(basename)\n if not m:\n raise ValueError(\n f\"Meta filename {basename!r} does not match expected pattern \"\n f\"'output_chunk\u003cNNNN>.meta.json'. Cannot derive chunk_id.\"\n )\n return m.group(1)\n\n\ndef _canonical_json(data):\n return json.dumps(data, sort_keys=True, separators=(',', ':'), ensure_ascii=False)\n\n\ndef meta_content_hash(data):\n \"\"\"SHA-256 of canonical JSON. Used by merge_meta.py to detect whether a\n given meta has already been applied to the glossary.\"\"\"\n return hashlib.sha256(_canonical_json(data).encode('utf-8')).hexdigest()\n\n\ndef _check_evidence(value, where, path):\n if not isinstance(value, str):\n raise ValueError(\n f\"Meta at {path}: 'evidence' in {where} must be a string, \"\n f\"got {type(value).__name__}\"\n )\n if len(value) > EVIDENCE_MAX_LEN:\n raise ValueError(\n f\"Meta at {path}: 'evidence' in {where} is {len(value)} chars; \"\n f\"limit is {EVIDENCE_MAX_LEN}. Quote a shorter excerpt.\"\n )\n\n\ndef _require_str(value, field, where, path):\n if not isinstance(value, str):\n raise ValueError(\n f\"Meta at {path}: {where} field {field!r} must be a string, \"\n f\"got {type(value).__name__}\"\n )\n\n\ndef _validate_array(data, key, path):\n val = data.get(key, [])\n if not isinstance(val, list):\n raise ValueError(\n f\"Meta at {path}: {key!r} must be a list, got {type(val).__name__}\"\n )\n return val\n\n\ndef validate_meta(data, path='\u003cmeta>'):\n \"\"\"Strict v1 validation. Raises ValueError with actionable messages.\"\"\"\n if not isinstance(data, dict):\n raise ValueError(\n f\"Meta at {path} must be a JSON object, got {type(data).__name__}\"\n )\n\n schema_version = data.get('schema_version')\n if schema_version != META_SCHEMA_VERSION:\n raise ValueError(\n f\"Meta at {path}: schema_version mismatch — expected \"\n f\"{META_SCHEMA_VERSION}, got {schema_version!r}.\"\n )\n\n extras = set(data.keys()) - VALID_TOP_LEVEL_KEYS\n if extras:\n # 'chunk_id' is the most likely offender; call it out specifically so\n # the fix is obvious.\n if 'chunk_id' in extras:\n raise ValueError(\n f\"Meta at {path}: 'chunk_id' field is not allowed in the meta \"\n f\"payload — chunk identity is derived from the filename. Remove it.\"\n )\n raise ValueError(\n f\"Meta at {path}: unknown top-level key(s) {sorted(extras)!r}. \"\n f\"Allowed keys: {sorted(VALID_TOP_LEVEL_KEYS)!r}.\"\n )\n\n for entity in _validate_array(data, 'new_entities', path):\n if not isinstance(entity, dict):\n raise ValueError(\n f\"Meta at {path}: each new_entities entry must be an object, \"\n f\"got {type(entity).__name__}\"\n )\n for required in ('source', 'target_proposal', 'evidence'):\n if required not in entity:\n raise ValueError(\n f\"Meta at {path}: new_entities entry missing required field \"\n f\"{required!r}\"\n )\n _require_str(entity['source'], 'source', 'new_entities', path)\n _require_str(entity['target_proposal'], 'target_proposal', 'new_entities', path)\n if 'category' in entity:\n _require_str(entity['category'], 'category', 'new_entities', path)\n _check_evidence(entity['evidence'], 'new_entities', path)\n\n for alias in _validate_array(data, 'alias_hypotheses', path):\n if not isinstance(alias, dict):\n raise ValueError(\n f\"Meta at {path}: each alias_hypotheses entry must be an object\"\n )\n for required in ('variant', 'may_be_alias_of_source', 'evidence'):\n if required not in alias:\n raise ValueError(\n f\"Meta at {path}: alias_hypotheses entry missing field {required!r}\"\n )\n _require_str(alias['variant'], 'variant', 'alias_hypotheses', path)\n _require_str(alias['may_be_alias_of_source'], 'may_be_alias_of_source',\n 'alias_hypotheses', path)\n _check_evidence(alias['evidence'], 'alias_hypotheses', path)\n\n for attr in _validate_array(data, 'attribute_hypotheses', path):\n if not isinstance(attr, dict):\n raise ValueError(\n f\"Meta at {path}: each attribute_hypotheses entry must be an object\"\n )\n for required in ('entity_source', 'attribute', 'value', 'confidence', 'evidence'):\n if required not in attr:\n raise ValueError(\n f\"Meta at {path}: attribute_hypotheses entry missing field {required!r}\"\n )\n _require_str(attr['entity_source'], 'entity_source', 'attribute_hypotheses', path)\n _require_str(attr['attribute'], 'attribute', 'attribute_hypotheses', path)\n _require_str(attr['value'], 'value', 'attribute_hypotheses', path)\n if attr['confidence'] not in VALID_ATTRIBUTE_CONFIDENCES:\n raise ValueError(\n f\"Meta at {path}: attribute_hypotheses 'confidence' must be one of \"\n f\"{list(VALID_ATTRIBUTE_CONFIDENCES)}, got {attr['confidence']!r}\"\n )\n _check_evidence(attr['evidence'], 'attribute_hypotheses', path)\n\n for s_idx, src in enumerate(_validate_array(data, 'used_term_sources', path)):\n if not isinstance(src, str):\n raise ValueError(\n f\"Meta at {path}: used_term_sources #{s_idx} must be a string, \"\n f\"got {type(src).__name__}\"\n )\n\n for conflict in _validate_array(data, 'conflicts', path):\n if not isinstance(conflict, dict):\n raise ValueError(\n f\"Meta at {path}: each conflicts entry must be an object\"\n )\n for required in ('entity_source', 'field', 'injected', 'observed_better', 'evidence'):\n if required not in conflict:\n raise ValueError(\n f\"Meta at {path}: conflicts entry missing field {required!r}\"\n )\n _require_str(conflict['entity_source'], 'entity_source', 'conflicts', path)\n _require_str(conflict['field'], 'field', 'conflicts', path)\n _require_str(conflict['injected'], 'injected', 'conflicts', path)\n _require_str(conflict['observed_better'], 'observed_better', 'conflicts', path)\n _check_evidence(conflict['evidence'], 'conflicts', path)\n\n\ndef load_meta(path):\n \"\"\"Load and strictly validate a meta file. Raises ValueError on any\n problem. Callers that want non-blocking behavior should wrap this in a\n quarantine wrapper (see merge_meta.py).\"\"\"\n if not os.path.exists(path):\n raise FileNotFoundError(f\"Meta not found: {path}\")\n with open(path, 'r', encoding='utf-8') as f:\n try:\n data = json.load(f)\n except json.JSONDecodeError as e:\n raise ValueError(f\"Meta at {path} is not valid JSON: {e}\") from e\n validate_meta(data, path)\n return data\n\n\ndef save_meta(path, data):\n \"\"\"Atomically write a meta file. Validates before writing.\"\"\"\n validate_meta(data, path)\n\n dirname = os.path.dirname(os.path.abspath(path)) or '.'\n fd, tmp_path = tempfile.mkstemp(dir=dirname, prefix='.meta-', suffix='.tmp')\n try:\n with os.fdopen(fd, 'w', encoding='utf-8') as f:\n json.dump(data, f, ensure_ascii=False, indent=2, sort_keys=True)\n f.write('\\n')\n os.replace(tmp_path, path)\n except Exception:\n if os.path.exists(tmp_path):\n try:\n os.unlink(tmp_path)\n except OSError:\n pass\n raise\n","content_type":"text/x-python; charset=utf-8","language":"python","size":9720,"content_sha256":"02aba18f0d75c9da2af603f1095b8d76d38f6f29748cfaf589918bfbd510039c"},{"filename":"scripts/run_state.py","content":"#!/usr/bin/env python3\n\"\"\"\nrun_state.py - Selective re-translation state for translate-book.\n\nThe state file records what glossary and source/output hashes were used for\neach translated chunk. Future runs can then decide which chunks need actual\nre-translation after glossary or source changes, and which existing outputs\nonly need their state recorded.\n\"\"\"\n\nimport argparse\nimport json\nimport os\nimport sys\nimport tempfile\nfrom datetime import datetime, timezone\nfrom pathlib import Path\n\nimport glossary as glossary_mod\nfrom manifest import file_hash, load_manifest, read_output_text\n\n\nRUN_STATE_VERSION = 1\nRUN_STATE_FILE = \"run_state.json\"\n\n\ndef _run_state_path(temp_dir):\n return os.path.join(temp_dir, RUN_STATE_FILE)\n\n\ndef _empty_state():\n return {\"version\": RUN_STATE_VERSION, \"chunks\": {}}\n\n\ndef load_run_state(temp_dir):\n path = _run_state_path(temp_dir)\n if not os.path.exists(path):\n return _empty_state()\n with open(path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n if data.get(\"version\") != RUN_STATE_VERSION:\n raise ValueError(\n f\"run_state.json version mismatch: expected {RUN_STATE_VERSION}, \"\n f\"got {data.get('version')!r}\"\n )\n chunks = data.get(\"chunks\")\n if not isinstance(chunks, dict):\n raise ValueError(\"run_state.json field 'chunks' must be an object\")\n return data\n\n\ndef save_run_state(temp_dir, state):\n path = _run_state_path(temp_dir)\n os.makedirs(temp_dir, exist_ok=True)\n fd, tmp_path = tempfile.mkstemp(prefix='.run_state.', suffix='.json', dir=temp_dir)\n try:\n with os.fdopen(fd, 'w', encoding='utf-8') as f:\n json.dump(state, f, ensure_ascii=False, indent=2, sort_keys=True)\n f.write('\\n')\n os.replace(tmp_path, path)\n except Exception:\n try:\n os.unlink(tmp_path)\n except OSError:\n pass\n raise\n\n\ndef _chunk_entries_from_manifest(temp_dir):\n manifest = load_manifest(temp_dir)\n if manifest:\n entries = []\n for chunk in sorted(manifest.get(\"chunks\", []), key=lambda c: c.get(\"order\", 0)):\n entries.append({\n \"id\": chunk[\"id\"],\n \"source_file\": chunk[\"source_file\"],\n \"output_file\": chunk[\"output_file\"],\n \"manifest_source_hash\": chunk.get(\"source_hash\", \"\"),\n \"order\": chunk.get(\"order\", 0),\n })\n return entries\n\n temp_path = Path(temp_dir)\n source_files = sorted(\n p for p in temp_path.glob('chunk*.md')\n if not p.name.startswith('output_')\n )\n return [\n {\n \"id\": p.stem,\n \"source_file\": p.name,\n \"output_file\": f\"output_{p.name}\",\n \"manifest_source_hash\": \"\",\n \"order\": i,\n }\n for i, p in enumerate(source_files, 1)\n ]\n\n\ndef _load_glossary(temp_dir):\n path = os.path.join(temp_dir, 'glossary.json')\n if not os.path.exists(path):\n return None, \"\", [], {}\n glossary = glossary_mod.load_glossary(path)\n glossary_hash = glossary_mod.glossary_hash(glossary)\n terms = glossary.get('terms', [])\n term_by_id = {t.get('id', t.get('source')): t for t in terms}\n return glossary, glossary_hash, terms, term_by_id\n\n\ndef _selected_terms_for_chunk(glossary, source_path):\n if glossary is None or not os.path.exists(source_path):\n return []\n text = Path(source_path).read_text(encoding='utf-8')\n return glossary_mod.select_terms_for_chunk(glossary, text)\n\n\ndef _term_ids_and_hashes(terms):\n ids = []\n hashes = {}\n for term in terms:\n term_id = term.get('id', term.get('source'))\n ids.append(term_id)\n hashes[term_id] = glossary_mod.term_hash(term)\n return ids, hashes\n\n\ndef _now_utc():\n return datetime.now(timezone.utc).replace(microsecond=0).isoformat()\n\n\ndef build_chunk_record(temp_dir, chunk_id):\n entries = {entry[\"id\"]: entry for entry in _chunk_entries_from_manifest(temp_dir)}\n if chunk_id not in entries:\n raise ValueError(f\"Unknown chunk id {chunk_id!r}\")\n\n entry = entries[chunk_id]\n source_path = os.path.join(temp_dir, entry[\"source_file\"])\n output_path = os.path.join(temp_dir, entry[\"output_file\"])\n if not os.path.exists(source_path):\n raise FileNotFoundError(f\"Source chunk not found: {source_path}\")\n if not os.path.exists(output_path):\n raise FileNotFoundError(f\"Output chunk not found: {output_path}\")\n if os.path.getsize(output_path) == 0:\n raise ValueError(f\"Output chunk is empty: {output_path}\")\n output_text = read_output_text(output_path)\n if output_text is None:\n raise ValueError(f\"Output chunk is not readable UTF-8 text: {output_path}\")\n if not output_text.strip():\n raise ValueError(f\"Output chunk is blank (whitespace-only): {output_path}\")\n\n glossary, glossary_hash, _, _ = _load_glossary(temp_dir)\n selected_terms = _selected_terms_for_chunk(glossary, source_path)\n entity_ids, entity_hashes = _term_ids_and_hashes(selected_terms)\n\n return {\n \"source_file\": entry[\"source_file\"],\n \"output_file\": entry[\"output_file\"],\n \"source_hash\": file_hash(source_path),\n \"output_hash\": file_hash(output_path),\n \"glossary_version_used\": glossary_hash,\n \"entity_ids_used\": entity_ids,\n \"entity_hashes_used\": entity_hashes,\n \"updated_at\": _now_utc(),\n }\n\n\ndef record_chunks(temp_dir, chunk_ids):\n state = load_run_state(temp_dir)\n recorded = []\n for chunk_id in chunk_ids:\n state[\"chunks\"][chunk_id] = build_chunk_record(temp_dir, chunk_id)\n recorded.append(chunk_id)\n save_run_state(temp_dir, state)\n return recorded\n\n\ndef _reason(item, code, detail=None):\n if detail is None:\n item[\"reasons\"].append(code)\n else:\n item[\"reasons\"].append({\"code\": code, \"detail\": detail})\n\n\ndef plan(temp_dir, retranslate_untracked=False):\n state = load_run_state(temp_dir)\n glossary, glossary_hash, _, _ = _load_glossary(temp_dir)\n\n result = {\n \"temp_dir\": temp_dir,\n \"glossary_hash\": glossary_hash,\n \"translation_chunk_ids\": [],\n \"record_only_chunk_ids\": [],\n \"unchanged_chunk_ids\": [],\n \"chunks\": [],\n \"decision_rules\": [\n \"missing_output_or_empty_output\",\n \"blank_or_unreadable_output\",\n \"manifest_source_hash_changed\",\n \"untracked_existing_output\",\n \"source_hash_changed_since_record\",\n \"glossary_term_selection_or_term_hash_changed\",\n ],\n \"record_update_rules\": [\n \"output_hash_changed_since_record\",\n ],\n }\n\n entries = _chunk_entries_from_manifest(temp_dir)\n records = state.get(\"chunks\", {})\n\n for entry in entries:\n chunk_id = entry[\"id\"]\n source_path = os.path.join(temp_dir, entry[\"source_file\"])\n output_path = os.path.join(temp_dir, entry[\"output_file\"])\n item = {\n \"chunk_id\": chunk_id,\n \"source_file\": entry[\"source_file\"],\n \"output_file\": entry[\"output_file\"],\n \"action\": \"unchanged\",\n \"reasons\": [],\n }\n\n if not os.path.exists(output_path):\n item[\"action\"] = \"translate\"\n _reason(item, \"missing_output\")\n elif os.path.getsize(output_path) == 0:\n item[\"action\"] = \"translate\"\n _reason(item, \"empty_output\")\n else:\n output_text = read_output_text(output_path)\n if output_text is None:\n item[\"action\"] = \"translate\"\n _reason(item, \"unreadable_output\")\n elif not output_text.strip():\n item[\"action\"] = \"translate\"\n _reason(item, \"blank_output\")\n\n current_source_hash = file_hash(source_path) if os.path.exists(source_path) else \"\"\n manifest_source_hash = entry.get(\"manifest_source_hash\", \"\")\n if item[\"action\"] == \"unchanged\" and manifest_source_hash:\n if current_source_hash != manifest_source_hash:\n item[\"action\"] = \"translate\"\n _reason(item, \"manifest_source_hash_changed\")\n\n record = records.get(chunk_id)\n if item[\"action\"] == \"unchanged\" and record is None:\n if retranslate_untracked:\n item[\"action\"] = \"translate\"\n _reason(item, \"untracked_existing_output\")\n else:\n item[\"action\"] = \"record\"\n _reason(item, \"untracked_existing_output\")\n\n if item[\"action\"] == \"unchanged\" and record is not None:\n if record.get(\"source_hash\") != current_source_hash:\n item[\"action\"] = \"translate\"\n _reason(item, \"source_hash_changed_since_record\")\n\n if item[\"action\"] == \"unchanged\" and record is not None:\n selected_terms = _selected_terms_for_chunk(glossary, source_path)\n current_entity_ids, current_entity_hashes = _term_ids_and_hashes(selected_terms)\n recorded_entity_ids = record.get(\"entity_ids_used\", [])\n recorded_entity_hashes = record.get(\"entity_hashes_used\", {})\n\n if current_entity_ids != recorded_entity_ids:\n item[\"action\"] = \"translate\"\n _reason(item, \"glossary_term_selection_changed\")\n else:\n changed_ids = [\n term_id for term_id in current_entity_ids\n if current_entity_hashes.get(term_id) != recorded_entity_hashes.get(term_id)\n ]\n if changed_ids:\n item[\"action\"] = \"translate\"\n _reason(item, \"glossary_term_hash_changed\", changed_ids)\n\n if item[\"action\"] == \"unchanged\" and record is not None:\n current_output_hash = file_hash(output_path) if os.path.exists(output_path) else \"\"\n if record.get(\"output_hash\") != current_output_hash:\n item[\"action\"] = \"record\"\n _reason(item, \"output_hash_changed_since_record\")\n\n if item[\"action\"] == \"translate\":\n result[\"translation_chunk_ids\"].append(chunk_id)\n elif item[\"action\"] == \"record\":\n result[\"record_only_chunk_ids\"].append(chunk_id)\n else:\n result[\"unchanged_chunk_ids\"].append(chunk_id)\n result[\"chunks\"].append(item)\n\n return result\n\n\ndef status(temp_dir):\n state = load_run_state(temp_dir)\n entries = _chunk_entries_from_manifest(temp_dir)\n planned = plan(temp_dir)\n return {\n \"temp_dir\": temp_dir,\n \"tracked_chunks\": len(state.get(\"chunks\", {})),\n \"source_chunks\": len(entries),\n \"translation_needed\": len(planned[\"translation_chunk_ids\"]),\n \"record_only_needed\": len(planned[\"record_only_chunk_ids\"]),\n \"unchanged\": len(planned[\"unchanged_chunk_ids\"]),\n }\n\n\ndef _print_json(data):\n print(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True))\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Track selective re-translation state\")\n sub = parser.add_subparsers(dest='cmd', required=True)\n\n p_plan = sub.add_parser('plan', help=\"Decide which chunks need translation or state recording\")\n p_plan.add_argument('temp_dir')\n p_plan.add_argument(\n '--retranslate-untracked',\n action='store_true',\n help=\"Treat existing outputs without run_state records as needing translation\",\n )\n\n p_record = sub.add_parser('record', help=\"Record one or more completed output chunks\")\n p_record.add_argument('temp_dir')\n p_record.add_argument('chunk_ids', nargs='+')\n\n p_record_all = sub.add_parser('record-all', help=\"Record every complete output chunk\")\n p_record_all.add_argument('temp_dir')\n\n p_status = sub.add_parser('status', help=\"Show run_state progress summary\")\n p_status.add_argument('temp_dir')\n\n args = parser.parse_args()\n\n try:\n if args.cmd == 'plan':\n _print_json(plan(args.temp_dir, retranslate_untracked=args.retranslate_untracked))\n elif args.cmd == 'record':\n _print_json({\"recorded_chunk_ids\": record_chunks(args.temp_dir, args.chunk_ids)})\n elif args.cmd == 'record-all':\n plan_data = plan(args.temp_dir)\n eligible = [\n item[\"chunk_id\"] for item in plan_data[\"chunks\"]\n if item[\"action\"] in (\"record\", \"unchanged\")\n ]\n _print_json({\"recorded_chunk_ids\": record_chunks(args.temp_dir, eligible)})\n elif args.cmd == 'status':\n _print_json(status(args.temp_dir))\n except Exception as e:\n print(f\"Error: {e}\", file=sys.stderr)\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":12734,"content_sha256":"7b3cb0bdc5e7c3c0af0d8f06517bcb9e0083bf3bb374bd978f8c972e1141d66b"},{"filename":"scripts/template_ebook.html","content":"\u003c!DOCTYPE html>\n\u003chtml lang=\"$lang$\">\n\u003chead>\n \u003cmeta charset=\"UTF-8\">\n \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n \u003ctitle>$title$\u003c/title>\n \u003cstyle>\n * {\n margin: 0;\n padding: 0;\n box-sizing: border-box;\n }\n\n html {\n font-size: 16px;\n scroll-behavior: smooth;\n }\n\n body {\n font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Helvetica Neue', Arial, 'Noto Sans', sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji';\n line-height: 1.7;\n color: #2c3e50;\n background-color: #ffffff;\n max-width: 800px;\n margin: 0 auto;\n padding: 2rem 1.5rem;\n font-size: 1rem;\n }\n\n /* 标题样式 */\n h1, h2, h3, h4, h5, h6 {\n font-weight: 600;\n margin-top: 2.5rem;\n margin-bottom: 1rem;\n color: #1a202c;\n line-height: 1.3;\n }\n\n h1 {\n font-size: 2.25rem;\n border-bottom: 3px solid #3182ce;\n padding-bottom: 0.75rem;\n margin-top: 0;\n margin-bottom: 2rem;\n }\n\n h2 {\n font-size: 1.875rem;\n border-bottom: 1px solid #e2e8f0;\n padding-bottom: 0.5rem;\n margin-top: 3rem;\n }\n\n h3 {\n font-size: 1.5rem;\n color: #2d3748;\n }\n\n h4 {\n font-size: 1.25rem;\n color: #4a5568;\n }\n\n h5 {\n font-size: 1.125rem;\n color: #4a5568;\n }\n\n h6 {\n font-size: 1rem;\n color: #718096;\n font-weight: 500;\n }\n\n /* 段落样式 */\n p {\n margin-bottom: 1.25rem;\n text-align: justify;\n word-break: break-word;\n }\n\n /* 链接样式 */\n a {\n color: #3182ce;\n text-decoration: none;\n transition: color 0.2s ease;\n }\n\n a:hover {\n color: #2c5282;\n text-decoration: underline;\n }\n\n /* 图片样式 - 居中显示 */\n img {\n max-width: 100%;\n height: auto;\n display: block;\n margin: 2rem auto;\n border-radius: 8px;\n box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);\n }\n\n /* 列表样式 */\n ul, ol {\n margin: 1.25rem 0;\n padding-left: 2rem;\n }\n\n li {\n margin-bottom: 0.5rem;\n }\n\n ul ul, ol ol, ul ol, ol ul {\n margin: 0.5rem 0;\n }\n\n /* 代码样式 */\n code {\n font-family: 'Fira Code', 'Monaco', 'Consolas', 'Ubuntu Mono', monospace;\n background-color: #f7fafc;\n color: #e53e3e;\n padding: 0.125rem 0.375rem;\n border-radius: 3px;\n font-size: 0.875rem;\n }\n\n pre {\n background-color: #f7fafc;\n border: 1px solid #e2e8f0;\n border-radius: 6px;\n padding: 1.25rem;\n margin: 1.5rem 0;\n overflow-x: auto;\n font-family: 'Fira Code', 'Monaco', 'Consolas', 'Ubuntu Mono', monospace;\n font-size: 0.875rem;\n line-height: 1.5;\n }\n\n pre code {\n background: none;\n color: inherit;\n padding: 0;\n border-radius: 0;\n font-size: inherit;\n }\n\n /* 引用样式 */\n blockquote {\n border-left: 4px solid #3182ce;\n margin: 1.5rem 0;\n padding: 1rem 1.5rem;\n background-color: #f7fafc;\n color: #4a5568;\n font-style: italic;\n border-radius: 0 6px 6px 0;\n }\n\n blockquote p:last-child {\n margin-bottom: 0;\n }\n\n /* 表格样式 */\n table {\n width: 100%;\n border-collapse: collapse;\n margin: 1.5rem 0;\n background-color: #ffffff;\n border-radius: 6px;\n overflow: hidden;\n box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);\n }\n\n th, td {\n padding: 0.75rem 1rem;\n text-align: left;\n border-bottom: 1px solid #e2e8f0;\n }\n\n th {\n background-color: #f7fafc;\n font-weight: 600;\n color: #2d3748;\n }\n\n tr:hover {\n background-color: #f7fafc;\n }\n\n /* 分隔线 */\n hr {\n border: none;\n border-top: 2px solid #e2e8f0;\n margin: 3rem 0;\n }\n\n /* 页面分隔器 */\n .page-separator {\n border-top: 2px solid #e2e8f0;\n margin: 3rem 0;\n text-align: center;\n position: relative;\n }\n\n .page-separator::after {\n content: \"• • •\";\n background-color: #ffffff;\n color: #a0aec0;\n padding: 0 1rem;\n position: relative;\n top: -0.75rem;\n }\n\n /* 强调样式 */\n strong, b {\n font-weight: 700;\n color: #1a202c;\n }\n\n em, i {\n font-style: italic;\n color: #4a5568;\n }\n\n /* 响应式设计 */\n @media (max-width: 768px) {\n body {\n padding: 1rem;\n font-size: 0.9rem;\n }\n\n h1 {\n font-size: 1.875rem;\n }\n\n h2 {\n font-size: 1.5rem;\n }\n\n h3 {\n font-size: 1.25rem;\n }\n\n h4 {\n font-size: 1.125rem;\n }\n\n table {\n font-size: 0.875rem;\n }\n\n th, td {\n padding: 0.5rem 0.75rem;\n }\n\n pre {\n padding: 1rem;\n font-size: 0.8rem;\n }\n\n blockquote {\n padding: 0.75rem 1rem;\n margin: 1rem 0;\n }\n }\n\n @media (max-width: 480px) {\n body {\n padding: 0.75rem;\n }\n\n h1, h2, h3, h4, h5, h6 {\n margin-top: 1.5rem;\n }\n\n ul, ol {\n padding-left: 1.5rem;\n }\n }\n\n /* 打印样式 */\n @media print {\n body {\n max-width: none;\n padding: 0;\n font-size: 12pt;\n line-height: 1.5;\n color: #000;\n }\n\n h1, h2, h3, h4, h5, h6 {\n page-break-after: avoid;\n }\n\n img {\n max-width: 100% !important;\n page-break-inside: avoid;\n }\n\n blockquote, pre {\n page-break-inside: avoid;\n }\n\n a {\n color: #000;\n text-decoration: underline;\n }\n }\n \u003c/style>\n\u003c/head>\n\u003cbody>\n $body$\n\u003c/body>\n\u003c/html>","content_type":"text/html; charset=utf-8","language":"markup","size":7158,"content_sha256":"7dca384c9a6e80e934d97c58166ac608eb3b80def1f3fa46f89746d787bab59b"},{"filename":"scripts/template.html","content":"\u003c!DOCTYPE html>\n\u003chtml lang=\"$lang$\">\n\u003chead>\n \u003cmeta charset=\"UTF-8\">\n \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n \u003ctitle>$title$\u003c/title>\n \u003cstyle>\n * {\n margin: 0;\n padding: 0;\n box-sizing: border-box;\n }\n\n html {\n font-size: 16px;\n scroll-behavior: smooth;\n }\n\n body {\n font-family: $body_font$;\n line-height: 1.8;\n color: #333333;\n background-color: #fefefe;\n max-width: 700px;\n margin: 0 auto;\n padding: 3rem 2rem;\n font-size: 1.1rem;\n text-align: justify;\n }\n\n /* 浮动目录按钮 */\n .toc-toggle {\n position: fixed;\n left: 20px;\n top: 50%;\n transform: translateY(-50%);\n z-index: 1000;\n background: rgba(96, 165, 250, 0.5);\n color: white;\n border: none;\n border-radius: 8px;\n width: 48px;\n height: 48px;\n font-size: 20px;\n cursor: pointer;\n box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);\n transition: all 0.3s ease;\n }\n\n .toc-toggle:hover {\n background: rgba(96, 165, 250, 0.8);\n transform: translateY(-50%) scale(1.1);\n }\n\n /* 浮动目录面板 */\n .toc-sidebar {\n position: fixed;\n left: -320px;\n top: 0;\n width: 300px;\n height: 100vh;\n background: #ffffff;\n border-right: 1px solid #e2e8f0;\n box-shadow: 4px 0 12px rgba(0, 0, 0, 0.1);\n z-index: 999;\n transition: left 0.3s ease;\n overflow-y: auto;\n padding: 2rem 1rem;\n }\n\n .toc-sidebar.active {\n left: 0;\n }\n\n .toc-sidebar h3 {\n font-size: 1.2rem;\n color: #1a202c;\n margin-bottom: 1rem;\n text-align: center;\n border-bottom: 2px solid #3182ce;\n padding-bottom: 0.5rem;\n }\n\n /* 目录区域 - 用于插入目录内容 */\n .toc-content {\n /* 目录内容将插入到这里 */\n }\n\n /* 遮罩层 */\n .toc-overlay {\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n background: rgba(0, 0, 0, 0.3);\n z-index: 998;\n opacity: 0;\n visibility: hidden;\n transition: all 0.3s ease;\n }\n\n .toc-overlay.active {\n opacity: 1;\n visibility: visible;\n }\n\n /* 简洁的目录样式 */\n .simple-toc ul {\n list-style: none;\n padding-left: 0;\n margin: 0;\n }\n\n .simple-toc ul ul {\n padding-left: 1rem;\n margin-top: 0.25rem;\n }\n\n .simple-toc li {\n margin-bottom: 0.25rem;\n }\n\n .simple-toc a {\n display: block;\n padding: 0.5rem 0.75rem;\n color: #4a5568;\n text-decoration: none;\n border-radius: 4px;\n transition: all 0.2s ease;\n font-size: 0.9rem;\n line-height: 1.4;\n }\n\n .simple-toc a:hover {\n background-color: #edf2f7;\n color: #3182ce;\n transform: translateX(4px);\n }\n\n /* 标题样式 */\n h1, h2, h3, h4, h5, h6 {\n font-family: $body_font$;\n color: #2d3748;\n margin-top: 2rem;\n margin-bottom: 1.2rem;\n line-height: 1.4;\n font-weight: normal;\n text-align: center;\n }\n\n h1 {\n font-size: 1.8rem;\n border-bottom: 2px solid #718096;\n padding-bottom: 0.8rem;\n margin-top: 0;\n margin-bottom: 2.5rem;\n }\n\n h2 {\n font-size: 1.5rem;\n border-bottom: 1px solid #cbd5e0;\n padding-bottom: 0.6rem;\n margin-top: 2.5rem;\n }\n\n h3 {\n font-size: 1.3rem;\n color: #4a5568;\n }\n\n h4 {\n font-size: 1.2rem;\n color: #4a5568;\n }\n\n h5 {\n font-size: 1.1rem;\n color: #4a5568;\n }\n\n h6 {\n font-size: 1rem;\n color: #718096;\n font-weight: normal;\n }\n\n /* 段落样式 */\n p {\n margin-bottom: 1.5rem;\n text-align: justify;\n word-break: break-word;\n text-indent: 2em;\n font-family: $body_font$;\n }\n\n /* 链接样式 */\n a {\n color: #3182ce;\n text-decoration: none;\n transition: color 0.2s ease;\n }\n\n a:hover {\n color: #2c5282;\n text-decoration: underline;\n }\n\n /* 图片样式 - 居中显示 */\n img {\n max-width: 100%;\n height: auto;\n display: block;\n margin: 2rem auto;\n border-radius: 8px;\n box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);\n }\n\n /* 列表样式 */\n ul, ol {\n margin: 1.25rem 0;\n padding-left: 2rem;\n }\n\n li {\n margin-bottom: 0.5rem;\n }\n\n ul ul, ol ol, ul ol, ol ul {\n margin: 0.5rem 0;\n }\n\n /* 代码样式 */\n code {\n font-family: 'Courier New', 'Monaco', 'Consolas', 'Ubuntu Mono', monospace;\n background-color: #f8f9fa;\n color: #d63384;\n padding: 0.2rem 0.4rem;\n border-radius: 3px;\n font-size: 0.9rem;\n }\n\n pre {\n background-color: #f8f9fa;\n border: 1px solid #e9ecef;\n border-radius: 6px;\n padding: 1.5rem;\n margin: 2rem 0;\n overflow-x: auto;\n font-family: 'Courier New', 'Monaco', 'Consolas', 'Ubuntu Mono', monospace;\n font-size: 0.9rem;\n line-height: 1.6;\n }\n\n pre code {\n background: none;\n color: inherit;\n padding: 0;\n border-radius: 0;\n font-size: inherit;\n }\n\n /* 引用样式 */\n blockquote {\n border-left: 4px solid #a0aec0;\n margin: 2rem 0;\n padding: 1.2rem 1.8rem;\n background-color: #f7fafc;\n color: #4a5568;\n font-style: italic;\n border-radius: 0 6px 6px 0;\n font-family: $body_font$;\n }\n\n blockquote p:last-child {\n margin-bottom: 0;\n }\n\n /* 表格样式 */\n table {\n width: 100%;\n border-collapse: collapse;\n margin: 1.5rem 0;\n background-color: #ffffff;\n border-radius: 6px;\n overflow: hidden;\n box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);\n }\n\n th, td {\n padding: 0.75rem 1rem;\n text-align: left;\n border-bottom: 1px solid #e2e8f0;\n }\n\n th {\n background-color: #f7fafc;\n font-weight: 600;\n color: #2d3748;\n }\n\n tr:hover {\n background-color: #f7fafc;\n }\n\n /* 分隔线 */\n hr {\n border: none;\n border-top: 2px solid #e2e8f0;\n margin: 3rem 0;\n }\n\n /* 页面分隔器 */\n .page-separator {\n border-top: 2px solid #e2e8f0;\n margin: 3rem 0;\n text-align: center;\n position: relative;\n }\n\n .page-separator::after {\n content: \"• • •\";\n background-color: #ffffff;\n color: #a0aec0;\n padding: 0 1rem;\n position: relative;\n top: -0.75rem;\n }\n\n /* 强调样式 */\n strong, b {\n font-weight: 700;\n color: #1a202c;\n }\n\n em, i {\n font-style: italic;\n color: #4a5568;\n }\n\n /* 目标标题高亮 */\n h1:target, h2:target, h3:target, h4:target, h5:target, h6:target {\n background-color: #fff3cd;\n padding: 1rem;\n border-radius: 6px;\n margin: 1.5rem 0;\n border-left: 4px solid #ffc107;\n animation: highlight 2s ease-in-out;\n }\n\n @keyframes highlight {\n 0% {\n background-color: #fff3cd;\n transform: scale(1.02);\n }\n 100% {\n background-color: transparent;\n transform: scale(1);\n }\n }\n\n /* 响应式设计 */\n @media (max-width: 768px) {\n body {\n padding: 1.5rem;\n font-size: 1rem;\n max-width: 100%;\n }\n\n .toc-toggle {\n left: 10px;\n width: 40px;\n height: 40px;\n font-size: 16px;\n }\n\n .toc-sidebar {\n width: 280px;\n left: -300px;\n padding: 1rem;\n }\n\n h1 {\n font-size: 1.6rem;\n }\n\n h2 {\n font-size: 1.4rem;\n }\n\n h3 {\n font-size: 1.2rem;\n }\n\n h4 {\n font-size: 1.1rem;\n }\n\n table {\n font-size: 0.9rem;\n }\n\n th, td {\n padding: 0.6rem 0.8rem;\n }\n\n pre {\n padding: 1.2rem;\n font-size: 0.85rem;\n }\n\n blockquote {\n padding: 1rem 1.2rem;\n margin: 1.5rem 0;\n }\n }\n\n @media (max-width: 480px) {\n body {\n padding: 1.2rem;\n font-size: 0.95rem;\n }\n\n .toc-sidebar {\n width: 260px;\n left: -280px;\n }\n\n h1, h2, h3, h4, h5, h6 {\n margin-top: 1.5rem;\n }\n\n ul, ol {\n padding-left: 1.5rem;\n }\n \n p {\n text-indent: 1.5em;\n }\n }\n\n /* 打印样式 */\n @media print {\n .toc-toggle,\n .toc-sidebar,\n .toc-overlay {\n display: none !important;\n }\n\n body {\n max-width: none;\n padding: 0;\n font-size: 12pt;\n line-height: 1.5;\n color: #000;\n }\n\n h1, h2, h3, h4, h5, h6 {\n page-break-after: avoid;\n }\n\n img {\n max-width: 100% !important;\n page-break-inside: avoid;\n }\n\n blockquote, pre {\n page-break-inside: avoid;\n }\n\n a {\n color: #000;\n text-decoration: underline;\n }\n }\n \u003c/style>\n\u003c/head>\n\u003cbody>\n \u003c!-- 浮动目录按钮 -->\n \u003cbutton class=\"toc-toggle\" onclick=\"toggleToc()\" title=\"打开/关闭目录\">\n ☰\n \u003c/button>\n\n \u003c!-- 浮动目录面板 -->\n \u003cdiv class=\"toc-sidebar\" id=\"tocSidebar\">\n \u003ch3>$toc_label$\u003c/h3>\n \u003cdiv class=\"toc-content simple-toc\">\n \u003c!-- 目录内容将插入到这里 -->\n \u003c/div>\n \u003c/div>\n\n \u003c!-- 遮罩层 -->\n \u003cdiv class=\"toc-overlay\" id=\"tocOverlay\" onclick=\"closeToc()\">\u003c/div>\n\n \u003c!-- 主要内容 -->\n $body$\n\n \u003cscript>\n function toggleToc() {\n const sidebar = document.getElementById('tocSidebar');\n const overlay = document.getElementById('tocOverlay');\n \n if (sidebar.classList.contains('active')) {\n closeToc();\n } else {\n openToc();\n }\n }\n\n function openToc() {\n const sidebar = document.getElementById('tocSidebar');\n const overlay = document.getElementById('tocOverlay');\n \n sidebar.classList.add('active');\n overlay.classList.add('active');\n document.body.style.overflow = 'hidden';\n }\n\n function closeToc() {\n const sidebar = document.getElementById('tocSidebar');\n const overlay = document.getElementById('tocOverlay');\n \n sidebar.classList.remove('active');\n overlay.classList.remove('active');\n document.body.style.overflow = 'auto';\n }\n\n // ESC键关闭目录\n document.addEventListener('keydown', function(e) {\n if (e.key === 'Escape') {\n closeToc();\n }\n });\n\n // 点击目录链接后自动关闭目录（移动端）\n document.addEventListener('click', function(e) {\n if (e.target.matches('.simple-toc a') && window.innerWidth \u003c= 768) {\n setTimeout(closeToc, 300);\n }\n });\n \u003c/script>\n\u003c/body>\n\u003c/html>","content_type":"text/html; charset=utf-8","language":"markup","size":13263,"content_sha256":"f3ec4e9c9df8d52525ec4d47db97a212b7b457b55b31ea145bcd2b53d212b8d6"}],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Book Translation Skill","type":"text"}]},{"type":"paragraph","content":[{"text":"You are a book translation assistant. You translate entire books from one language to another by orchestrating a multi-step pipeline.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Workflow","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"1. Collect Parameters","type":"text"}]},{"type":"paragraph","content":[{"text":"Determine the following from the user's message:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"file_path","type":"text","marks":[{"type":"strong"}]},{"text":": Path to the input file (PDF, DOCX, EPUB, or Markdown) — REQUIRED","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"target_lang","type":"text","marks":[{"type":"strong"}]},{"text":": Target language code (default: ","type":"text"},{"text":"zh","type":"text","marks":[{"type":"code_inline"}]},{"text":") — e.g. zh, en, ja, ko, fr, de, es","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"concurrency","type":"text","marks":[{"type":"strong"}]},{"text":": Number of parallel sub-agents per batch (default: ","type":"text"},{"text":"8","type":"text","marks":[{"type":"code_inline"}]},{"text":")","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"temp_root","type":"text","marks":[{"type":"strong"}]},{"text":": Optional directory under which ","type":"text"},{"text":"{filename}_temp/","type":"text","marks":[{"type":"code_inline"}]},{"text":" should be created","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"epub_cover","type":"text","marks":[{"type":"strong"}]},{"text":": Optional explicit cover image path for EPUB output","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"export_name","type":"text","marks":[{"type":"strong"}]},{"text":": Optional filename stem for user-facing output aliases","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"custom_instructions","type":"text","marks":[{"type":"strong"}]},{"text":": Any additional translation instructions from the user (optional)","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"If the file path is not provided, ask the user.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"2. Preprocess — Convert to Markdown Chunks","type":"text"}]},{"type":"paragraph","content":[{"text":"Run the conversion script to produce chunks:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/convert.py \"\u003cfile_path>\" --olang \"\u003ctarget_lang>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"If the user provided ","type":"text"},{"text":"temp_root","type":"text","marks":[{"type":"code_inline"}]},{"text":", add ","type":"text"},{"text":"--temp-root \"\u003ctemp_root>\"","type":"text","marks":[{"type":"code_inline"}]},{"text":". The temp directory leaf name remains ","type":"text"},{"text":"{filename}_temp/","type":"text","marks":[{"type":"code_inline"}]},{"text":"; only the parent directory changes.","type":"text"}]},{"type":"paragraph","content":[{"text":"For PDFs, Calibre reflows text by coordinate heuristics, which shatters math formulas, flattens tables, and interleaves multi-column layouts before translation starts. If the PDF is an academic or technical document and a layout-aware parser is already installed — or the user asks for one — extract the PDF with it first, then pass the resulting Markdown file to ","type":"text"},{"text":"convert.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" instead of the PDF. Markdown input (","type":"text"},{"text":".md","type":"text","marks":[{"type":"code_inline"}]},{"text":" / ","type":"text"},{"text":".markdown","type":"text","marks":[{"type":"code_inline"}]},{"text":") skips Calibre:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"MinerU >= 4: ","type":"text"},{"text":"mineru-kit parse \"\u003cfile_path>\" -o \"\u003cname>.md\"","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Marker: ","type":"text"},{"text":"marker_single \"\u003cfile_path>\" --output_dir \"\u003cdir>\"","type":"text","marks":[{"type":"code_inline"}]},{"text":", then use ","type":"text"},{"text":"\u003cdir>/\u003cname>/\u003cname>.md","type":"text","marks":[{"type":"code_inline"}]}]}]}]},{"type":"paragraph","content":[{"text":"Keep the PDF's file name stem for the Markdown file, because the temp directory is named after it. These CLIs change between versions; check the parser's ","type":"text"},{"text":"--help","type":"text","marks":[{"type":"code_inline"}]},{"text":" if a flag is rejected. Do not install a parser without the user's consent, because they download large models. Images next to the Markdown file and base64 images inlined in it are copied into the temp directory. ","type":"text"},{"text":"--strip-page-numbers","type":"text","marks":[{"type":"code_inline"}]},{"text":" does not apply to Markdown input.","type":"text"}]},{"type":"paragraph","content":[{"text":"Page-number lines are detected and dropped for PDF input only; EPUB and DOCX keep every standalone number (chapter, verse or exercise numbers). If an EPUB prints its page numbers as text and they show up in the chunks, delete the temp directory before translating and re-run with ","type":"text"},{"text":"--detect-page-numbers","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"This creates a ","type":"text"},{"text":"{filename}_temp/","type":"text","marks":[{"type":"code_inline"}]},{"text":" directory containing:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"input.html","type":"text","marks":[{"type":"code_inline"}]},{"text":" (Calibre input only), ","type":"text"},{"text":"input.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — intermediate files","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"chunk0001.md","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"chunk0002.md","type":"text","marks":[{"type":"code_inline"}]},{"text":", ... — source chunks for translation","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"manifest.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" — chunk manifest for tracking and validation","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"source_fingerprint.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" — SHA-256 identity of the source bytes this temp dir was built from","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"config.txt","type":"text","marks":[{"type":"code_inline"}]},{"text":" — pipeline configuration with metadata","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"If ","type":"text"},{"text":"convert.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" aborts because the temp dir was created from different source bytes, do not reuse it — delete the temp directory or pass a fresh ","type":"text"},{"text":"--temp-root","type":"text","marks":[{"type":"code_inline"}]},{"text":", then re-run. Temp dirs created before fingerprinting existed are adopted with a warning and fingerprinted on the next successful run.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"3. Discover Source Chunks","type":"text"}]},{"type":"paragraph","content":[{"text":"Use Glob to find all source chunks:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":""},"content":[{"text":"Glob: {filename}_temp/chunk*.md","type":"text"}]},{"type":"paragraph","content":[{"text":"Exclude ","type":"text"},{"text":"output_chunk*.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" from the source list. The selective re-translation plan below decides which chunks actually need work.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"3.5. Build Glossary (term consistency)","type":"text"}]},{"type":"paragraph","content":[{"text":"A separate sub-agent translates each chunk with a fresh context. Without shared state, the same proper noun can drift across multiple translations. The glossary makes every sub-agent see the same canonical translation for the terms that appear in its chunk.","type":"text"}]},{"type":"paragraph","content":[{"text":"If ","type":"text"},{"text":"\u003ctemp_dir>/glossary.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" already exists, skip the rebuild — re-running the skill must not overwrite a hand-edited glossary. To force a rebuild, delete the file.","type":"text"}]},{"type":"paragraph","content":[{"text":"Otherwise:","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Sample chunks","type":"text","marks":[{"type":"strong"}]},{"text":": read ","type":"text"},{"text":"chunk0001.md","type":"text","marks":[{"type":"code_inline"}]},{"text":", the last chunk, and 3 evenly-spaced middle chunks. If ","type":"text"},{"text":"chunk_count \u003c 5","type":"text","marks":[{"type":"code_inline"}]},{"text":", sample all of them.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Extract terms","type":"text","marks":[{"type":"strong"}]},{"text":": from the samples, identify proper nouns and recurring domain terms that need consistent translation across the book — typically people, places, organizations, technical concepts. Translate each into the target language. Skip generic vocabulary that any translator would render the same way.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Write ","type":"text","marks":[{"type":"strong"}]},{"text":"glossary.json","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" in the temp dir, matching this v2 schema:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"json"},"content":[{"text":"{\n \"version\": 2,\n \"terms\": [\n {\"id\": \"Manhattan\", \"source\": \"Manhattan\", \"target\": \"曼哈顿\",\n \"category\": \"place\", \"aliases\": [], \"gender\": \"unknown\",\n \"confidence\": \"medium\", \"frequency\": 0,\n \"evidence_refs\": [], \"notes\": \"\"}\n ],\n \"high_frequency_top_n\": 20,\n \"applied_meta_hashes\": {}\n}","type":"text"}]},{"type":"paragraph","content":[{"text":"Existing v1 ","type":"text"},{"text":"glossary.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" files are auto-upgraded to v2 on first load. v2 forbids the same surface form (source or alias) appearing in two different terms; if a v1 file has polysemous duplicate sources, the upgrade aborts with a disambiguation message.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Count frequencies","type":"text","marks":[{"type":"strong"}]},{"text":" by running:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/glossary.py count-frequencies \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"This scans every ","type":"text"},{"text":"chunk*.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" (excluding ","type":"text"},{"text":"output_chunk*.md","type":"text","marks":[{"type":"code_inline"}]},{"text":"), updates each term's ","type":"text"},{"text":"frequency","type":"text","marks":[{"type":"code_inline"}]},{"text":" field, and writes back atomically.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The glossary is hand-editable. If the user edits a ","type":"text"},{"text":"target","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"aliases","type":"text","marks":[{"type":"code_inline"}]},{"text":", or ","type":"text"},{"text":"category","type":"text","marks":[{"type":"code_inline"}]},{"text":" field after a partial run, the run-state planner in the next step will re-translate only chunks whose recorded term set or term hashes are affected.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"3.7. Plan Selective Re-translation","type":"text"}]},{"type":"paragraph","content":[{"text":"Run:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/run_state.py plan \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"If the user explicitly asks to apply glossary edits to outputs produced before ","type":"text"},{"text":"run_state.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" existed, add ","type":"text"},{"text":"--retranslate-untracked","type":"text","marks":[{"type":"code_inline"}]},{"text":"; otherwise keep the default so old temp dirs remain resumable without mass re-translation.","type":"text"}]},{"type":"paragraph","content":[{"text":"Capture stdout JSON:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"translation_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" — chunks to translate in this run.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"record_only_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" — existing valid outputs that need ","type":"text"},{"text":"run_state.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" records but do not need translation.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"unchanged_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" — existing outputs already consistent with the current source chunks and glossary.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"If ","type":"text"},{"text":"record_only_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" is non-empty, record them before launching sub-agents:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/run_state.py record \"\u003ctemp_dir>\" chunk0001 chunk0002 ...","type":"text"}]},{"type":"paragraph","content":[{"text":"Use ","type":"text"},{"text":"translation_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" as the work queue for Step 4. If it is empty, skip to Step 5.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"4. Parallel Translation with Sub-Agents","type":"text"}]},{"type":"paragraph","content":[{"text":"Each chunk gets its own independent sub-agent","type":"text","marks":[{"type":"strong"}]},{"text":" (1 chunk = 1 sub-agent = 1 fresh context). This prevents context accumulation and output truncation.","type":"text"}]},{"type":"paragraph","content":[{"text":"Launch chunks in batches to respect API rate limits:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Each batch: up to ","type":"text"},{"text":"concurrency","type":"text","marks":[{"type":"code_inline"}]},{"text":" sub-agents in parallel (default: 8)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Wait for the current batch to complete before launching the next","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Spawn each sub-agent with the following task.","type":"text","marks":[{"type":"strong"}]},{"text":" Use whatever sub-agent/background-agent mechanism your runtime provides (e.g. the Agent tool, sessions_spawn, or equivalent).","type":"text"}]},{"type":"paragraph","content":[{"text":"The output file is ","type":"text"},{"text":"output_","type":"text","marks":[{"type":"code_inline"}]},{"text":" prefixed to the source filename: ","type":"text"},{"text":"chunk0001.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" → ","type":"text"},{"text":"output_chunk0001.md","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]},{"type":"blockquote","content":[{"type":"paragraph","content":[{"text":"Translate the file ","type":"text"},{"text":"\u003ctemp_dir>/chunk\u003cNNNN>.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" to {TARGET_LANGUAGE} and write the result to ","type":"text"},{"text":"\u003ctemp_dir>/output_chunk\u003cNNNN>.md","type":"text","marks":[{"type":"code_inline"}]},{"text":". Follow the translation rules below. Output only the translated content — no commentary.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Each sub-agent receives:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"The single chunk file it is responsible for","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"The temp directory path","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"The target language","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"The translation prompt (see below)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"A per-chunk term table (see \"Term table assembly\" below)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Read-only neighboring chunk excerpts (see \"Neighbor context assembly\" below)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Any custom instructions","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Term table assembly","type":"text","marks":[{"type":"strong"}]},{"text":" — before spawning a sub-agent, run:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/glossary.py print-terms-for-chunk \"\u003ctemp_dir>\" \"chunk\u003cNNNN>.md\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Capture stdout. The CLI emits a 3-column markdown table (","type":"text"},{"text":"原文 | 别名 | 译文","type":"text","marks":[{"type":"code_inline"}]},{"text":") of every term that either appears in this chunk (by source OR any alias) OR is in the top-N most-frequent terms book-wide. Inject the table as ","type":"text"},{"text":"{TERM_TABLE}","type":"text","marks":[{"type":"code_inline"}]},{"text":" in rule #13 of the translation prompt. ","type":"text"},{"text":"If stdout is empty (no glossary, or no relevant terms), omit rule #13 from this chunk's prompt entirely","type":"text","marks":[{"type":"strong"}]},{"text":" — do not leave a dangling ","type":"text"},{"text":"{TERM_TABLE}","type":"text","marks":[{"type":"code_inline"}]},{"text":" placeholder.","type":"text"}]},{"type":"paragraph","content":[{"text":"Neighbor context assembly","type":"text","marks":[{"type":"strong"}]},{"text":" — before spawning a sub-agent, run:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/chunk_context.py \"\u003ctemp_dir>\" \"chunk\u003cNNNN>.md\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Capture stdout. The CLI emits prompt-ready read-only excerpts: the last ~300 characters of the previous chunk and the first ~300 characters of the next chunk when those files exist. Inject this block as ","type":"text"},{"text":"{NEIGHBOR_CONTEXT}","type":"text","marks":[{"type":"code_inline"}]},{"text":". If stdout is empty, omit the neighbor-context block entirely. The sub-agent must not translate neighboring excerpts or copy them into the output; they are only for pronoun, gender, and entity-resolution context.","type":"text"}]},{"type":"paragraph","content":[{"text":"Each sub-agent's task","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Read the source chunk file (e.g. ","type":"text"},{"text":"chunk0001.md","type":"text","marks":[{"type":"code_inline"}]},{"text":")","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Translate the content following the translation rules below","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Write the translated content to ","type":"text"},{"text":"output_chunk0001.md","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Write observations to ","type":"text"},{"text":"output_chunk0001.meta.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" matching the schema below. ","type":"text"},{"text":"Non-blocking","type":"text","marks":[{"type":"strong"}]},{"text":" — leave fields empty if unsure; do not invent entities. Always emit the file (even if all arrays are empty), because its presence + content hash is how the main agent tracks whether feedback was already merged.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Sub-agent meta schema","type":"text","marks":[{"type":"strong"}]},{"text":" (","type":"text"},{"text":"output_chunk\u003cNNNN>.meta.json","type":"text","marks":[{"type":"code_inline"}]},{"text":"):","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"json"},"content":[{"text":"{\n \"schema_version\": 1,\n \"new_entities\": [\n {\"source\": \"Taig\", \"target_proposal\": \"泰格\", \"category\": \"person\",\n \"evidence\": \"\u003c≤200-char quote from the chunk>\"}\n ],\n \"alias_hypotheses\": [\n {\"variant\": \"Taig\", \"may_be_alias_of_source\": \"Tai\",\n \"evidence\": \"\u003c≤200-char quote>\"}\n ],\n \"attribute_hypotheses\": [\n {\"entity_source\": \"Tai\", \"attribute\": \"gender\", \"value\": \"male\",\n \"confidence\": \"high\", \"evidence\": \"\u003c≤200-char quote>\"}\n ],\n \"used_term_sources\": [\"Tai\", \"Manhattan\"],\n \"conflicts\": [\n {\"entity_source\": \"Tai\", \"field\": \"target\", \"injected\": \"泰\",\n \"observed_better\": \"太一\", \"evidence\": \"\u003c≤200-char quote>\"}\n ]\n}","type":"text"}]},{"type":"paragraph","content":[{"text":"Do NOT include a ","type":"text","marks":[{"type":"strong"}]},{"text":"chunk_id","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" field","type":"text","marks":[{"type":"strong"}]},{"text":" — chunk identity is derived from the filename. Putting it in the payload creates a hallucination hole and validation will reject the file.","type":"text"}]},{"type":"paragraph","content":[{"text":"The meta file is read by the main agent later and merged into ","type":"text"},{"text":"glossary.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" (see ","type":"text"},{"text":"merge_meta.py","type":"text","marks":[{"type":"code_inline"}]},{"text":"). Sub-agents should fill the schema honestly: cite real quotes from the chunk, never invent entities to \"look productive\". An empty meta is a perfectly valid output.","type":"text"}]},{"type":"paragraph","content":[{"text":"IMPORTANT","type":"text","marks":[{"type":"strong"}]},{"text":": Each sub-agent translates exactly ONE chunk and writes the result directly to the output file. No START/END markers needed.","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Translation Prompt for Sub-Agents","type":"text"}]},{"type":"paragraph","content":[{"text":"Include this translation prompt in each sub-agent's instructions (replace ","type":"text"},{"text":"{TARGET_LANGUAGE}","type":"text","marks":[{"type":"code_inline"}]},{"text":" with the actual language name, e.g. \"Chinese\"):","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"paragraph","content":[{"text":"请翻译markdown文件为 {TARGET_LANGUAGE}. IMPORTANT REQUIREMENTS:","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"严格保持 Markdown 格式不变，包括标题、链接、图片引用等","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"仅翻译文字内容，保留所有 Markdown 语法和文件名","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"数学公式（行内 ","type":"text"},{"text":"$...$","type":"text","marks":[{"type":"code_inline"}]},{"text":"、公式块 ","type":"text"},{"text":"$...$","type":"text","marks":[{"type":"code_inline"}]},{"text":"）内的 LaTeX 原样保留，不要翻译、改写或删除其中任何字符","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"表格（Markdown ","type":"text"},{"text":"| ... |","type":"text","marks":[{"type":"code_inline"}]},{"text":" 表格或 HTML ","type":"text"},{"text":"\u003ctable>","type":"text","marks":[{"type":"code_inline"}]},{"text":"）保持行列结构不变，只翻译单元格中的文字","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"脚注：正文中的引用标记 ","type":"text"},{"text":"[^12]","type":"text","marks":[{"type":"code_inline"}]},{"text":" 与行首的定义标记 ","type":"text"},{"text":"[^12]:","type":"text","marks":[{"type":"code_inline"}]},{"text":" 原样保留，不要改编号、不要删除、不要移动位置，只翻译定义后面的注释文字；定义的续段保持行首四个空格的缩进","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"删除空链接、不必要的字符和如: 行末的'\\'。页码已由 convert.py 上游处理，不要再删除独立的数字行（可能是年份 1984、章节编号、引用编号等正文内容）。","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"保证格式和语义准确翻译内容自然流畅","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"只输出翻译后的正文内容，不要有任何说明、提示、注释或对话内容。","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"表达清晰简洁，不要使用复杂的句式。请严格按顺序翻译，不要跳过任何内容。","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"必须保留所有图片引用，包括：","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"所有 ","type":"text"},{"text":"![alt](path)","type":"text","marks":[{"type":"code_inline"}]},{"text":" 格式的图片引用必须完整保留","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"图片文件名和路径不要修改（如 ","type":"text"},{"text":"media/image-001.png","type":"text","marks":[{"type":"code_inline"}]},{"text":"）","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"图片alt文本可以翻译，但必须保留图片引用结构","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"不要删除、过滤或忽略任何图片相关内容","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"图片引用示例：","type":"text"},{"text":"![Figure 1: Data Flow](media/image-001.png)","type":"text","marks":[{"type":"code_inline"}]},{"text":" -> ","type":"text"},{"text":"![图1：数据流](media/image-001.png)","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"原始 HTML 标签（如 ","type":"text","marks":[{"type":"strong"}]},{"text":"\u003cimg alt=\"...\" />","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":"、","type":"text","marks":[{"type":"strong"}]},{"text":"\u003ca title=\"...\">","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":"）必须保持合法","type":"text","marks":[{"type":"strong"}]},{"text":"：翻译 ","type":"text"},{"text":"alt","type":"text","marks":[{"type":"code_inline"}]},{"text":"、","type":"text"},{"text":"title","type":"text","marks":[{"type":"code_inline"}]},{"text":" 等属性值内部文本时，下列字符会破坏 HTML 结构，必须替换为安全形式（仅适用于","type":"text"},{"text":"原始 HTML 标签的属性值内部","type":"text","marks":[{"type":"strong"}]},{"text":"；普通 Markdown 正文、代码块、URL 不要主动转义）：","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"字符","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"在属性值内的危险","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"替换为","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"\"","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"闭合 ","type":"text"},{"text":"attr=\"...\"","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"目标语言合适的弯引号（如中文 ","type":"text"},{"text":"“","type":"text","marks":[{"type":"code_inline"}]},{"text":" ","type":"text"},{"text":"”","type":"text","marks":[{"type":"code_inline"}]},{"text":"）或 ","type":"text"},{"text":""","type":"text","marks":[{"type":"code_inline"}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"'","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"闭合 ","type":"text"},{"text":"attr='...'","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"目标语言合适的弯引号（如中文 ","type":"text"},{"text":"‘","type":"text","marks":[{"type":"code_inline"}]},{"text":" ","type":"text"},{"text":"’","type":"text","marks":[{"type":"code_inline"}]},{"text":"）或 ","type":"text"},{"text":"'","type":"text","marks":[{"type":"code_inline"}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"\u003c","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"被解析为新标签","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"<","type":"text","marks":[{"type":"code_inline"}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":">","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"被解析为标签结束","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":">","type":"text","marks":[{"type":"code_inline"}]}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"&","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"被解析为实体起始（除非已是 ","type":"text"},{"text":"&xxx;","type":"text","marks":[{"type":"code_inline"}]},{"text":"）","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"&","type":"text","marks":[{"type":"code_inline"}]}]}]}]}]},{"type":"paragraph","content":[{"text":"不要修改 ","type":"text"},{"text":"src","type":"text","marks":[{"type":"code_inline"}]},{"text":"、","type":"text"},{"text":"href","type":"text","marks":[{"type":"code_inline"}]},{"text":" 等结构性属性的值，只翻译可见文本属性（","type":"text"},{"text":"alt","type":"text","marks":[{"type":"code_inline"}]},{"text":"、","type":"text"},{"text":"title","type":"text","marks":[{"type":"code_inline"}]},{"text":"）。","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"错误示例：","type":"text"},{"text":"alt=\"爱丽丝拿着标着\"喝我\"的瓶子\"","type":"text","marks":[{"type":"code_inline"}]},{"text":" ← 内层英文 ","type":"text"},{"text":"\"","type":"text","marks":[{"type":"code_inline"}]},{"text":" 把外层 alt 撑断了","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"正确示例：","type":"text"},{"text":"alt=\"爱丽丝拿着标着“喝我”的瓶子\"","type":"text","marks":[{"type":"code_inline"}]},{"text":" 或 ","type":"text"},{"text":"alt=\"爱丽丝拿着标着"喝我"的瓶子\"","type":"text","marks":[{"type":"code_inline"}]}]}]}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"智能识别和处理多级标题，按照以下规则添加markdown标记：","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"主标题（书名、章节名等）使用 # 标记","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"一级标题（大节标题）使用 ## 标记","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"二级标题（小节标题）使用 ### 标记","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"三级标题（子标题）使用 #### 标记","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"四级及以下标题使用 ##### 标记","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"标题识别规则：","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"独立成行的较短文本（通常少于50字符）","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"具有总结性或概括性的语句","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"在文档结构中起到分隔和组织作用的文本","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"字体大小明显不同或有特殊格式的文本","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"数字编号开头的章节文本（如 \"1.1 概述\"、\"第三章\"等）","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"标题层级判断：","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"根据上下文和内容重要性判断标题层级","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"章节类标题通常为高层级（# 或 ##）","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"小节、子节标题依次降级（### #### #####）","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"保持同一文档内标题层级的一致性","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"注意事项：","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"不要过度添加标题标记，只对真正的标题文本添加","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"正文段落不要添加标题标记","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"如果原文已有markdown标题标记，保持其层级结构","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"{CUSTOM_INSTRUCTIONS if provided}","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"术语一致性：以下术语必须严格使用指定译法，不要自行变换。表格中\"原文\"列","type":"text"},{"text":"或\"别名\"列","type":"text","marks":[{"type":"strong"}]},{"text":"任一形式出现在正文中时，都必须翻译为\"译文\"列对应的形式。","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"{TERM_TABLE}","type":"text"}]},{"type":"paragraph","content":[{"text":"邻居上下文（只读，不要翻译，不要写入输出，只用于判断代词、性别、别名和跨 chunk 指代；为空则省略）:","type":"text"}]},{"type":"paragraph","content":[{"text":"{NEIGHBOR_CONTEXT}","type":"text"}]},{"type":"paragraph","content":[{"text":"markdown文件正文:","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":3},"content":[{"text":"4.5. Merge Sub-Agent Meta Into Glossary (after each batch)","type":"text"}]},{"type":"paragraph","content":[{"text":"Each sub-agent emitted an ","type":"text"},{"text":"output_chunk\u003cNNNN>.meta.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" alongside its translated chunk. After every batch completes, first record the completed chunk outputs in ","type":"text"},{"text":"run_state.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" while the glossary is still the one used for that batch, then merge observations into the canonical glossary so subsequent batches see an enriched glossary.","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Record successfully translated chunks from this batch before mutating the glossary:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/run_state.py record \"\u003ctemp_dir>\" chunk0001 chunk0002 ...","type":"text"}]},{"type":"paragraph","content":[{"text":"If this fails, fix the missing/empty output or state error before continuing.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Run prepare-merge:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/merge_meta.py prepare-merge \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Capture stdout JSON. It contains four arrays:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"auto_apply","type":"text","marks":[{"type":"code_inline"}]},{"text":" — new entities with no glossary collision and unanimous (target, category) across all proposing chunks.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"decisions_needed","type":"text","marks":[{"type":"code_inline"}]},{"text":" — items requiring main-agent judgment. Each has ","type":"text"},{"text":"id","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"kind","type":"text","marks":[{"type":"code_inline"}]},{"text":", an ","type":"text"},{"text":"options","type":"text","marks":[{"type":"code_inline"}]},{"text":" array, and the data needed to pick. Kinds:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"alias","type":"text","marks":[{"type":"code_inline"}]},{"text":" — ","type":"text"},{"text":"{variant, candidate_source, evidence}","type":"text","marks":[{"type":"code_inline"}]},{"text":". Choices: ","type":"text"},{"text":"yes_alias","type":"text","marks":[{"type":"code_inline"}]},{"text":" / ","type":"text"},{"text":"no_separate_entity","type":"text","marks":[{"type":"code_inline"}]},{"text":" / ","type":"text"},{"text":"skip","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"conflict","type":"text","marks":[{"type":"code_inline"}]},{"text":" — ","type":"text"},{"text":"{entity_source, field, current, proposed, evidence}","type":"text","marks":[{"type":"code_inline"}]},{"text":". Choices: ","type":"text"},{"text":"keep_current","type":"text","marks":[{"type":"code_inline"}]},{"text":" / ","type":"text"},{"text":"accept_proposed","type":"text","marks":[{"type":"code_inline"}]},{"text":" / ","type":"text"},{"text":"record_in_notes","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"new_entity_existing_alias","type":"text","marks":[{"type":"code_inline"}]},{"text":" — sub-agents propose ","type":"text"},{"text":"proposed_source","type":"text","marks":[{"type":"code_inline"}]},{"text":" as a new entity, but it's already someone's alias. ","type":"text"},{"text":"{proposed_source, currently_alias_of, promoted_variants: [{target_proposal, category, evidence, evidence_chunks}, ...]}","type":"text","marks":[{"type":"code_inline"}]},{"text":". Choices: one ","type":"text"},{"text":"use_variant_N","type":"text","marks":[{"type":"code_inline"}]},{"text":" per distinct (target, category) promotion variant (promote ","type":"text"},{"text":"proposed_source","type":"text","marks":[{"type":"code_inline"}]},{"text":" to standalone with that target+category, removing it from the host's aliases) / ","type":"text"},{"text":"keep_as_alias","type":"text","marks":[{"type":"code_inline"}]},{"text":" / ","type":"text"},{"text":"skip","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"existing_entity_conflict","type":"text","marks":[{"type":"code_inline"}]},{"text":" — sub-agents proposed a (target, category) for ","type":"text"},{"text":"entity_source","type":"text","marks":[{"type":"code_inline"}]},{"text":" that differs from the canonical. Multiple distinct differing proposals all get exposed. ","type":"text"},{"text":"{entity_source, current_target, current_category, proposed_variants: [{target_proposal, category, evidence, evidence_chunks}, ...]}","type":"text","marks":[{"type":"code_inline"}]},{"text":". Choices: ","type":"text"},{"text":"keep_current","type":"text","marks":[{"type":"code_inline"}]},{"text":" / one ","type":"text"},{"text":"use_variant_N","type":"text","marks":[{"type":"code_inline"}]},{"text":" per competing proposal (overwrites both target AND category, stamps the prior values into notes) / ","type":"text"},{"text":"record_in_notes","type":"text","marks":[{"type":"code_inline"}]},{"text":" (canonical unchanged; every proposed variant gets logged to notes).","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"alias_or_new_entity","type":"text","marks":[{"type":"code_inline"}]},{"text":" — ","type":"text"},{"text":"variant","type":"text","marks":[{"type":"code_inline"}]},{"text":" has multiple competing options that can't all coexist under v2's surface-form uniqueness rule. Triggered when (a) ","type":"text"},{"text":"variant","type":"text","marks":[{"type":"code_inline"}]},{"text":" was proposed both as a new standalone entity AND as an alias of one or more candidates, OR (b) ","type":"text"},{"text":"variant","type":"text","marks":[{"type":"code_inline"}]},{"text":" was proposed as an alias of two or more different candidates with no standalone competitor. ","type":"text"},{"text":"{variant, alias_candidates: [{candidate_source, evidence, evidence_chunks}, ...], standalone_variants: [{target_proposal, category, evidence, evidence_chunks}, ...]}","type":"text","marks":[{"type":"code_inline"}]},{"text":". Choices: one ","type":"text"},{"text":"use_alias_N","type":"text","marks":[{"type":"code_inline"}]},{"text":" per candidate (attach as alias of that candidate), one ","type":"text"},{"text":"use_standalone_N","type":"text","marks":[{"type":"code_inline"}]},{"text":" per competing standalone proposal (add as standalone with that target+category), or ","type":"text"},{"text":"skip","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"conflicting_new_entity_proposals","type":"text","marks":[{"type":"code_inline"}]},{"text":" — ","type":"text"},{"text":"{source, variants: [{target_proposal, category, evidence, evidence_chunks}, ...]}","type":"text","marks":[{"type":"code_inline"}]},{"text":". Choices: ","type":"text"},{"text":"use_variant_0","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"use_variant_1","type":"text","marks":[{"type":"code_inline"}]},{"text":", ..., ","type":"text"},{"text":"skip","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"consumed_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" — every meta file scanned this round (regardless of whether it produced a finding). These hashes get recorded in ","type":"text"},{"text":"applied_meta_hashes","type":"text","marks":[{"type":"code_inline"}]},{"text":" on apply.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"malformed_meta_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" — meta files that failed validation. Quarantined: not consumed, not crashing the run. Surface them in your batch progress.","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"If ","type":"text","marks":[{"type":"strong"}]},{"text":"consumed_chunk_ids","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" is empty","type":"text","marks":[{"type":"strong"}]},{"text":" → nothing was scanned; skip to Step 5.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"If ","type":"text","marks":[{"type":"strong"}]},{"text":"consumed_chunk_ids","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" is non-empty but both ","type":"text","marks":[{"type":"strong"}]},{"text":"auto_apply","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" and ","type":"text","marks":[{"type":"strong"}]},{"text":"decisions_needed","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" are empty","type":"text","marks":[{"type":"strong"}]},{"text":" → still pipe ","type":"text"},{"text":"{\"auto_apply\": [], \"decisions\": [], \"consumed_chunk_ids\": [...]}","type":"text","marks":[{"type":"code_inline"}]},{"text":" into ","type":"text"},{"text":"apply-merge","type":"text","marks":[{"type":"code_inline"}]},{"text":" so the hashes get recorded. ","type":"text"},{"text":"Skipping this is the bug","type":"text","marks":[{"type":"strong"}]},{"text":" — no-op metas would re-scan forever otherwise.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Otherwise, resolve each decision","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Read its evidence quotes inline.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Pick one option from its ","type":"text"},{"text":"options","type":"text","marks":[{"type":"code_inline"}]},{"text":" array.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Build a ","type":"text"},{"text":"decisions","type":"text","marks":[{"type":"code_inline"}]},{"text":" entry that round-trips the original decision plus your choice. The entry MUST include the original ","type":"text"},{"text":"kind","type":"text","marks":[{"type":"code_inline"}]},{"text":" and (for ","type":"text"},{"text":"conflicting_new_entity_proposals","type":"text","marks":[{"type":"code_inline"}]},{"text":") the ","type":"text"},{"text":"variants","type":"text","marks":[{"type":"code_inline"}]},{"text":" array, so apply-merge can validate and act:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"json"},"content":[{"text":"{\"id\": \"d1\", \"kind\": \"alias\", \"variant\": \"Taig\", \"candidate_source\": \"Tai\", \"choice\": \"yes_alias\"}","type":"text"}]}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Pipe the decisions JSON into apply-merge:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"echo '{\"auto_apply\": [...], \"decisions\": [...], \"consumed_chunk_ids\": [...]}' \\\n | python3 {baseDir}/scripts/merge_meta.py apply-merge \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Surface the summary JSON (","type":"text"},{"text":"auto_applied","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"decisions_resolved","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"consumed_chunks","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"errors","type":"text","marks":[{"type":"code_inline"}]},{"text":") in your batch progress message.","type":"text"}]},{"type":"paragraph","content":[{"text":"apply-merge is transactional.","type":"text","marks":[{"type":"strong"}]},{"text":" If any decision is malformed (wrong choice for kind, missing fields, references a non-existent entity), the entire batch aborts with a non-zero exit and stderr details — no glossary mutation, no hashes recorded. On non-zero exit, fix the offending decision and re-pipe; ","type":"text"},{"text":"prepare-merge","type":"text","marks":[{"type":"code_inline"}]},{"text":" will surface the same proposals because nothing was consumed.","type":"text"}]},{"type":"paragraph","content":[{"text":"Decision order in the input list is not significant.","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"},{"text":"apply-merge","type":"text","marks":[{"type":"code_inline"}]},{"text":" internally dispatches entity-creating decisions before alias-attaching ones, so ","type":"text"},{"text":"yes_alias","type":"text","marks":[{"type":"code_inline"}]},{"text":" decisions whose candidate is created by another decision in the same batch (a ","type":"text"},{"text":"use_standalone_N","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"use_variant_N","type":"text","marks":[{"type":"code_inline"}]},{"text":", or ","type":"text"},{"text":"promote_to_separate_entity","type":"text","marks":[{"type":"code_inline"}]},{"text":") succeed regardless of the order you pass them in. Alias chains (e.g. ","type":"text"},{"text":"Taighi → Taig","type":"text","marks":[{"type":"code_inline"}]},{"text":" where ","type":"text"},{"text":"Taig → Tai","type":"text","marks":[{"type":"code_inline"}]},{"text":" is also a pending alias decision) resolve via a fixed-point loop within the alias-attacher pass — you don't need to topo-sort or sequence chained aliases manually.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"On a fresh run after a previous interrupted batch, ","type":"text"},{"text":"prepare-merge","type":"text","marks":[{"type":"code_inline"}]},{"text":" will pick up any meta files left behind. Don't manually delete them.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"5. Verify Completeness and Retry","type":"text"}]},{"type":"paragraph","content":[{"text":"After all batches complete, use Glob to check that every source chunk has a corresponding output file.","type":"text"}]},{"type":"paragraph","content":[{"text":"If any are missing, retry them — each missing chunk as its own sub-agent. Maximum 2 attempts per chunk (initial + 1 retry).","type":"text"}]},{"type":"paragraph","content":[{"text":"Also read ","type":"text"},{"text":"manifest.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" and verify:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Every chunk id has a corresponding output file","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"No output file is empty (0 bytes) or blank (whitespace-only)","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Then run the meta-merge observability snapshot:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/merge_meta.py status \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Also run the selective re-translation state snapshot:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/run_state.py status \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Surface a one-line summary in the verification report:","type":"text"}]},{"type":"blockquote","content":[{"type":"paragraph","content":[{"text":"Translated chunks: 50 • Meta files: 48 found / 47 consumed • Malformed: 1 (chunk0099 — see stderr) • Chunks missing meta: chunk0017, chunk0042","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Severity rules (none of these fail the run — meta is non-blocking):","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"unmerged_meta_files > 0","type":"text","marks":[{"type":"code_inline"}]},{"text":" after Step 4.5 ran → bug, flag prominently. Resume should have caught this.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"malformed_meta_files > 0","type":"text","marks":[{"type":"code_inline"}]},{"text":" → sub-agent emitted invalid meta; print chunk_ids and a \"fix the file by hand and re-run if you want this chunk's feedback merged\" note.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"meta_files_found \u003c translated_chunks","type":"text","marks":[{"type":"code_inline"}]},{"text":" → sub-agent-compliance issue (some chunks didn't emit meta at all). Print missing chunk_ids.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Report any chunks that failed translation after retry.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"6. Translate Book Title","type":"text"}]},{"type":"paragraph","content":[{"text":"Read ","type":"text"},{"text":"config.txt","type":"text","marks":[{"type":"code_inline"}]},{"text":" from the temp directory to get the ","type":"text"},{"text":"original_title","type":"text","marks":[{"type":"code_inline"}]},{"text":" field. If it is missing (e.g. Markdown input with neither front matter nor a leading ","type":"text"},{"text":"#","type":"text","marks":[{"type":"code_inline"}]},{"text":" title), use the source file name instead.","type":"text"}]},{"type":"paragraph","content":[{"text":"Translate the title to the target language. For Chinese, wrap in 书名号: ","type":"text"},{"text":"《translated_title》","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"7. Post-process — Merge and Build","type":"text"}]},{"type":"paragraph","content":[{"text":"Run the build script with the translated title:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/merge_and_build.py --temp-dir \"\u003ctemp_dir>\" --title \"\u003ctranslated_title>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"If the user provided ","type":"text"},{"text":"epub_cover","type":"text","marks":[{"type":"code_inline"}]},{"text":", add ","type":"text"},{"text":"--cover \"\u003cepub_cover>\"","type":"text","marks":[{"type":"code_inline"}]},{"text":". If the user provided ","type":"text"},{"text":"export_name","type":"text","marks":[{"type":"code_inline"}]},{"text":", add ","type":"text"},{"text":"--export-name \"\u003cexport_name>\"","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"Keep the intermediate files. The chunks, ","type":"text"},{"text":"manifest.json","type":"text","marks":[{"type":"code_inline"}]},{"text":", and ","type":"text"},{"text":"run_state.json","type":"text","marks":[{"type":"code_inline"}]},{"text":" are what make spot fixes and selective re-translation possible after the build (see Step 9). Add ","type":"text"},{"text":"--cleanup","type":"text","marks":[{"type":"code_inline"}]},{"text":" only when the user explicitly asks to delete intermediates, and tell them first that it permanently deletes ","type":"text"},{"text":"chunk*.md","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"output_chunk*.md","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"input.html","type":"text","marks":[{"type":"code_inline"}]},{"text":", and ","type":"text"},{"text":"input.md","type":"text","marks":[{"type":"code_inline"}]},{"text":": after that, fixing a single chunk means converting and translating the whole book again.","type":"text"}]},{"type":"paragraph","content":[{"text":"The script reads ","type":"text"},{"text":"output_lang","type":"text","marks":[{"type":"code_inline"}]},{"text":" from ","type":"text"},{"text":"config.txt","type":"text","marks":[{"type":"code_inline"}]},{"text":" automatically. Optional overrides: ","type":"text"},{"text":"--lang","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"--author","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"This produces in the temp directory:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"output.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — merged translated markdown","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"book.html","type":"text","marks":[{"type":"code_inline"}]},{"text":" — web version with floating TOC","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"book_doc.html","type":"text","marks":[{"type":"code_inline"}]},{"text":" — ebook version","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"book.docx","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"book.epub","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"book.pdf","type":"text","marks":[{"type":"code_inline"}]},{"text":" — format conversions (requires Calibre)","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"8. Report Results","type":"text"}]},{"type":"paragraph","content":[{"text":"Tell the user:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Where the output files are located","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"How many chunks were translated","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"The translated title","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"List generated output files with sizes","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Any format generation failures","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"9. Spot Fixes After the Build","type":"text"}]},{"type":"paragraph","content":[{"text":"If the user finds problems in a few places after the build, fix the affected chunks rather than ","type":"text"},{"text":"output.md","type":"text","marks":[{"type":"code_inline"}]},{"text":": whenever a chunk changes, the build re-merges ","type":"text"},{"text":"output.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" from the chunks and discards edits made there.","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Find the chunk: search ","type":"text"},{"text":"\u003ctemp_dir>/output_chunk*.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" for the passage.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Either edit ","type":"text"},{"text":"output_chunk\u003cNNNN>.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" directly, or have it re-translated.","type":"text"}]},{"type":"paragraph","content":[{"text":"Edit.","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"},{"text":"record","type":"text","marks":[{"type":"code_inline"}]},{"text":" stamps the chunk as matching the current source and glossary, so first check what the planner still expects from it:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/run_state.py plan \"\u003ctemp_dir>\"","type":"text"}]},{"type":"paragraph","content":[{"text":"If chunk\u003cNNNN> appears in ","type":"text"},{"text":"chunks[]","type":"text","marks":[{"type":"code_inline"}]},{"text":" with action ","type":"text"},{"text":"translate","type":"text","marks":[{"type":"code_inline"}]},{"text":", recording would hide that. Handle it by reason:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"glossary_term_hash_changed","type":"text","marks":[{"type":"code_inline"}]},{"text":" (its ","type":"text"},{"text":"detail","type":"text","marks":[{"type":"code_inline"}]},{"text":" lists the term ids) or ","type":"text"},{"text":"glossary_term_selection_changed","type":"text","marks":[{"type":"code_inline"}]},{"text":": bring those terms in line with ","type":"text"},{"text":"glossary.py print-terms-for-chunk \"\u003ctemp_dir>\" \"chunk\u003cNNNN>.md\"","type":"text","marks":[{"type":"code_inline"}]},{"text":" in the same edit.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"manifest_source_hash_changed","type":"text","marks":[{"type":"code_inline"}]},{"text":": the source chunk was edited after splitting. Re-run the Step 2 ","type":"text"},{"text":"convert.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" command first, leaving out page-number flags such as ","type":"text"},{"text":"--strip-page-numbers","type":"text","marks":[{"type":"code_inline"}]},{"text":": once chunks exist, those flags abort and suggest deleting the cached files or the whole temp dir, which would also delete the translations. Without them it keeps the existing chunks and writes their new hashes to ","type":"text"},{"text":"manifest.json","type":"text","marks":[{"type":"code_inline"}]},{"text":". Skip this and the merge stops with \"Source changed since splitting\" even after re-translating. Plan again: the chunk is then queued with ","type":"text"},{"text":"source_hash_changed_since_record","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"source_hash_changed_since_record","type":"text","marks":[{"type":"code_inline"}]},{"text":" or any other reason: re-translate the chunk instead of editing it.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Then record the edited output:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python3 {baseDir}/scripts/run_state.py record \"\u003ctemp_dir>\" chunk\u003cNNNN>","type":"text"}]},{"type":"paragraph","content":[{"text":"Re-translate.","type":"text","marks":[{"type":"strong"}]},{"text":" Delete ","type":"text"},{"text":"output_chunk\u003cNNNN>.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" and run Steps 3.7–4.5 again. The planner re-checks the whole book, so ","type":"text"},{"text":"translation_chunk_ids","type":"text","marks":[{"type":"code_inline"}]},{"text":" can also list chunks the user did not mention, typically ones whose glossary terms changed after they were translated (","type":"text"},{"text":"glossary_term_selection_changed","type":"text","marks":[{"type":"code_inline"}]},{"text":" or ","type":"text"},{"text":"glossary_term_hash_changed","type":"text","marks":[{"type":"code_inline"}]},{"text":" in the plan's ","type":"text"},{"text":"chunks[].reasons","type":"text","marks":[{"type":"code_inline"}]},{"text":"). Translate only the chunks the user asked for; if the plan lists others, show the user those chunk ids with their reasons and ask before re-translating them.","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Re-run the Step 7 build command with the same flags. It re-merges because the chunk is newer than ","type":"text"},{"text":"output.md","type":"text","marks":[{"type":"code_inline"}]},{"text":", then regenerates the HTML and the other formats.","type":"text"}]}]}]}]},"metadata":{"date":"2026-11-02","name":"translate-book","author":"@skillopedia","source":{"stars":2176,"repo_name":"translate-book","origin_url":"https://github.com/deusyu/translate-book/blob/HEAD/SKILL.md","repo_owner":"deusyu","body_sha256":"cb9c6da1df9c1486a0beae6a7c21e7cc914d686bb6ab1fb793eefcba7144d44a","cluster_key":"e20b1c40cf76c0889300e464d94ddf6ff756de2abaa330c120c89e4180cc3c75","clean_bundle":{"format":"clean-skill-bundle-v1","source":"deusyu/translate-book/SKILL.md","text_bytes":241720,"source_commit":"8a2d71cba25c514bfa6b6d6191b1fe21c0fe3229","skills_sh_rank":8778,"attachment_count":12,"text_attachments":12,"attachment_storage":"skill_attachments","binary_attachments":0},"cluster_size":1,"skill_md_path":"SKILL.md","import_metadata":{"date":"2026-11-02","author":"@skillopedia","version":"v1","category":"documents-office","category_label":"Documents"},"exact_dupes_collapsed_into_this":0},"version":"v1","category":"documents-office","metadata":{"openclaw":{"homepage":"https://github.com/deusyu/translate-book","requires":{"bins":["python3","pandoc","ebook-convert"],"anyBins":["calibre","ebook-convert"]}}},"import_tag":"skills-sh-top20k-2026-11","description":"Translate books (PDF/DOCX/EPUB) into any language using parallel sub-agents. Converts input -> Markdown chunks -> translated chunks -> HTML/DOCX/EPUB/PDF.","allowed-tools":"Read, Write, Edit, Bash, Glob, Grep, Agent, AskUserQuestion"}},"renderedAt":1793666431816}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.