Source code for document2md.converter

"""Shell out to mineru to OCR a PDF or a set of scanned page images into
Markdown, then rewrite its raw HTML table fallback into Markdown tables (see
document2md.tables).

Reuses an already-running mineru-api server via MINERU_API_URL when one is
set (see document2md.mineru_server), so a batch of documents shares one warm
server instead of each call starting and stopping its own.
"""
import contextlib
import os
import shutil
import subprocess
import tempfile
from pathlib import Path

from document2md.mineru_server import ENV_VAR as MINERU_API_URL_ENV_VAR
from document2md.mineru_server import MineruServer
from document2md.tables import html_tables_to_markdown

# Some real DOF editions run to hundreds of pages with heavy table content,
# and mineru has been observed to stall indefinitely on a single page/table
# in such cases rather than just running slowly. This bounds how long any
# one conversion is allowed to run before we give up on it.
DEFAULT_TIMEOUT_SECONDS = 3600


[docs] def _mineru_out_dir(md_path: Path, keep_mineru_output: bool): """Context manager yielding the directory mineru should write its raw output (Markdown, layout/model JSON, rendered PDFs...) into. Normally that output is only a stepping stone to the Markdown this module returns, so it lives in a throwaway temp dir. When `keep_mineru_output` is set, callers debugging a bad conversion need to see that raw output instead of it vanishing with the temp dir, so it is written to `<md stem>_mineru/` next to `md_path` and left there.""" if keep_mineru_output: out_dir = md_path.parent / f"{md_path.stem}_mineru" out_dir.mkdir(parents=True, exist_ok=True) return contextlib.nullcontext(str(out_dir)) return tempfile.TemporaryDirectory()
[docs] def _require_mineru() -> None: """Raise RuntimeError with an actionable message if the `mineru` CLI isn't on PATH, instead of letting `subprocess.run` fail opaquely.""" if shutil.which("mineru") is None: raise RuntimeError( "'mineru' is required for the mineru backend but isn't installed. " 'Install it with: pip install "document2md[mineru]"' )
[docs] def _run_mineru(input_path: Path, tmp_out: str, timeout: float) -> tuple[str, Path | None]: """Run mineru's pipeline backend on one input file (PDF or image) inside the given temp directory. Returns the raw Markdown text and the directory of any figures mineru extracted (or None if there were none). Reuses the MINERU_API_URL server if set (see mineru_server.MineruServer), so batch conversions don't reload models once per document.""" api_url = os.environ.get(MINERU_API_URL_ENV_VAR) cmd = ["mineru", "-o", tmp_out, "-p", str(input_path), "-b", "pipeline"] if api_url: cmd += ["--api-url", api_url] subprocess.run(cmd, check=True, timeout=timeout) auto_dir = Path(tmp_out) / input_path.stem / "auto" md_text = (auto_dir / f"{input_path.stem}.md").read_text(encoding="utf-8") images_src = auto_dir / "images" if images_src.is_dir() and any(images_src.iterdir()): return md_text, images_src return md_text, None
[docs] def convert_to_markdown( pdf_path: Path, md_path: Path, timeout: float = DEFAULT_TIMEOUT_SECONDS, keep_mineru_output: bool = False, ) -> None: """Convert a PDF to Markdown using mineru's pipeline backend. mineru auto-detects which parts of the document need OCR internally, so this handles both born-digital and scanned DOF editions, and produces much cleaner Markdown (real paragraphs, accurate headings) than plain text-layer extraction — at the cost of being far slower (mineru runs real layout/OCR models even on already-digital text). Complex tables, which mineru emits as raw HTML, are rewritten to Markdown tables (see tables.html_tables_to_markdown) so the output is Markdown all the way through. If MINERU_API_URL is set (see mineru_server.MineruServer), reuses that already-running server instead of letting mineru spin up (and tear down) its own temporary one for this single call — this is what lets a batch of documents avoid reloading the models once per document. `keep_mineru_output` writes mineru's raw output (its own Markdown, layout JSON, rendered layout/model PDFs...) to `<md stem>_mineru/` next to `md_path` instead of a temp dir that disappears at the end of the call — useful when a conversion looks wrong and mineru's own read of the page is the first thing worth inspecting. Raises subprocess.TimeoutExpired if mineru doesn't finish within `timeout` seconds — callers doing batch work should catch this and skip the offending document rather than let one stuck conversion block an entire run.""" _require_mineru() with _mineru_out_dir(md_path, keep_mineru_output) as tmp_out: md_text, images_src = _run_mineru(pdf_path, tmp_out, timeout) if images_src is not None: images_dirname = f"{pdf_path.stem}_images" shutil.copytree( images_src, md_path.parent / images_dirname, dirs_exist_ok=True ) md_text = md_text.replace("](images/", f"]({images_dirname}/") md_path.write_text(html_tables_to_markdown(md_text), encoding="utf-8")
[docs] def convert_images_to_markdown( image_paths: list[Path], md_path: Path, timeout: float = DEFAULT_TIMEOUT_SECONDS, keep_mineru_output: bool = False, ) -> None: """Convert one or more scanned page images into a single Markdown document. This is the OCR path for notes that come as scanned pages rather than digital text — e.g. the JPEGs downloaded by dofjson's download_nota_imagenes(). Each image is OCR'd with the same mineru pipeline backend as convert_to_markdown() and the results are concatenated in the order given (page order), so a note spanning several pages becomes one continuous Markdown file. Complex tables mineru emits as raw HTML are rewritten to Markdown tables (see tables.html_tables_to_markdown), like convert_to_markdown(). The page images typically hold more than the note of interest (a page can start or end mid-note); callers wanting only one note should slice the result afterwards — see cutter.cut_markdown_by_titles(), or use BatchConverter, which does the cut inline. Figures mineru extracts from a page are copied next to the output under `<md stem>_images/<image stem>/` (namespaced per page so figures from different pages can't collide), and their Markdown references rewritten to match. Raises subprocess.TimeoutExpired like convert_to_markdown(). `keep_mineru_output`, like in convert_to_markdown(), keeps mineru's raw per-page output instead of discarding it — each page lands in its own `<md stem>_mineru/<image stem>/` subdirectory, mirroring how the extracted figures are namespaced per page. A note spanning several pages OCRs each page with a separate `mineru` invocation; left alone, that means reloading mineru's layout/OCR models once per page. When MINERU_API_URL isn't already set (i.e. no caller is already running a batch server across multiple notes) and there's more than one page, this starts a MineruServer for the duration of the loop so all of this note's pages share one already-warm server instead of each page starting and stopping its own.""" _require_mineru() image_paths = [Path(p) for p in image_paths] if not image_paths: raise ValueError("convert_images_to_markdown requires at least one image path") images_dirname = f"{md_path.stem}_images" parts = [] needs_server = len(image_paths) > 1 and MINERU_API_URL_ENV_VAR not in os.environ server = MineruServer() if needs_server else contextlib.nullcontext() with server, _mineru_out_dir(md_path, keep_mineru_output) as tmp_out: for image_path in image_paths: md_text, images_src = _run_mineru(image_path, tmp_out, timeout) if images_src is not None: page_dest = md_path.parent / images_dirname / image_path.stem shutil.copytree(images_src, page_dest, dirs_exist_ok=True) md_text = md_text.replace( "](images/", f"]({images_dirname}/{image_path.stem}/" ) parts.append(md_text.strip()) md_path.write_text( html_tables_to_markdown("\n\n".join(parts)) + "\n", encoding="utf-8" )