"""Shell out to mineru to OCR a PDF or a set of scanned page images into
Markdown, then rewrite its raw HTML table fallback into Markdown tables (see
document2md.tables).
Reuses an already-running mineru-api server via MINERU_API_URL when one is
set (see document2md.mineru_server), so a batch of documents shares one warm
server instead of each call starting and stopping its own.
"""
import contextlib
import os
import shutil
import subprocess
import tempfile
from pathlib import Path
from document2md.mineru_server import ENV_VAR as MINERU_API_URL_ENV_VAR
from document2md.mineru_server import MineruServer
from document2md.tables import html_tables_to_markdown
# Some real DOF editions run to hundreds of pages with heavy table content,
# and mineru has been observed to stall indefinitely on a single page/table
# in such cases rather than just running slowly. This bounds how long any
# one conversion is allowed to run before we give up on it.
DEFAULT_TIMEOUT_SECONDS = 3600
[docs]
def _mineru_out_dir(md_path: Path, keep_mineru_output: bool):
"""Context manager yielding the directory mineru should write its raw
output (Markdown, layout/model JSON, rendered PDFs...) into.
Normally that output is only a stepping stone to the Markdown this module
returns, so it lives in a throwaway temp dir. When `keep_mineru_output` is
set, callers debugging a bad conversion need to see that raw output
instead of it vanishing with the temp dir, so it is written to
`<md stem>_mineru/` next to `md_path` and left there."""
if keep_mineru_output:
out_dir = md_path.parent / f"{md_path.stem}_mineru"
out_dir.mkdir(parents=True, exist_ok=True)
return contextlib.nullcontext(str(out_dir))
return tempfile.TemporaryDirectory()
[docs]
def _require_mineru() -> None:
"""Raise RuntimeError with an actionable message if the `mineru` CLI
isn't on PATH, instead of letting `subprocess.run` fail opaquely."""
if shutil.which("mineru") is None:
raise RuntimeError(
"'mineru' is required for the mineru backend but isn't installed. "
'Install it with: pip install "document2md[mineru]"'
)
[docs]
def _run_mineru(input_path: Path, tmp_out: str, timeout: float) -> tuple[str, Path | None]:
"""Run mineru's pipeline backend on one input file (PDF or image) inside
the given temp directory. Returns the raw Markdown text and the directory
of any figures mineru extracted (or None if there were none).
Reuses the MINERU_API_URL server if set (see mineru_server.MineruServer),
so batch conversions don't reload models once per document."""
api_url = os.environ.get(MINERU_API_URL_ENV_VAR)
cmd = ["mineru", "-o", tmp_out, "-p", str(input_path), "-b", "pipeline"]
if api_url:
cmd += ["--api-url", api_url]
subprocess.run(cmd, check=True, timeout=timeout)
auto_dir = Path(tmp_out) / input_path.stem / "auto"
md_text = (auto_dir / f"{input_path.stem}.md").read_text(encoding="utf-8")
images_src = auto_dir / "images"
if images_src.is_dir() and any(images_src.iterdir()):
return md_text, images_src
return md_text, None
[docs]
def convert_to_markdown(
pdf_path: Path,
md_path: Path,
timeout: float = DEFAULT_TIMEOUT_SECONDS,
keep_mineru_output: bool = False,
) -> None:
"""Convert a PDF to Markdown using mineru's pipeline backend.
mineru auto-detects which parts of the document need OCR internally, so
this handles both born-digital and scanned DOF editions, and produces
much cleaner Markdown (real paragraphs, accurate headings) than plain
text-layer extraction — at the cost of being far slower (mineru runs
real layout/OCR models even on already-digital text).
Complex tables, which mineru emits as raw HTML, are rewritten to Markdown
tables (see tables.html_tables_to_markdown) so the output is Markdown all
the way through.
If MINERU_API_URL is set (see mineru_server.MineruServer), reuses that
already-running server instead of letting mineru spin up (and tear down)
its own temporary one for this single call — this is what lets a batch
of documents avoid reloading the models once per document.
`keep_mineru_output` writes mineru's raw output (its own Markdown, layout
JSON, rendered layout/model PDFs...) to `<md stem>_mineru/` next to
`md_path` instead of a temp dir that disappears at the end of the call —
useful when a conversion looks wrong and mineru's own read of the page is
the first thing worth inspecting.
Raises subprocess.TimeoutExpired if mineru doesn't finish within
`timeout` seconds — callers doing batch work should catch this and skip
the offending document rather than let one stuck conversion block an
entire run."""
_require_mineru()
with _mineru_out_dir(md_path, keep_mineru_output) as tmp_out:
md_text, images_src = _run_mineru(pdf_path, tmp_out, timeout)
if images_src is not None:
images_dirname = f"{pdf_path.stem}_images"
shutil.copytree(
images_src, md_path.parent / images_dirname, dirs_exist_ok=True
)
md_text = md_text.replace("](images/", f"]({images_dirname}/")
md_path.write_text(html_tables_to_markdown(md_text), encoding="utf-8")
[docs]
def convert_images_to_markdown(
image_paths: list[Path],
md_path: Path,
timeout: float = DEFAULT_TIMEOUT_SECONDS,
keep_mineru_output: bool = False,
) -> None:
"""Convert one or more scanned page images into a single Markdown document.
This is the OCR path for notes that come as scanned pages rather than
digital text — e.g. the JPEGs downloaded by dofjson's
download_nota_imagenes(). Each image is OCR'd with the same mineru
pipeline backend as convert_to_markdown() and the results are
concatenated in the order given (page order), so a note spanning several
pages becomes one continuous Markdown file.
Complex tables mineru emits as raw HTML are rewritten to Markdown tables
(see tables.html_tables_to_markdown), like convert_to_markdown().
The page images typically hold more than the note of interest (a page can
start or end mid-note); callers wanting only one note should slice the
result afterwards — see cutter.cut_markdown_by_titles(), or use
BatchConverter, which does the cut inline.
Figures mineru extracts from a page are copied next to the output under
`<md stem>_images/<image stem>/` (namespaced per page so figures from
different pages can't collide), and their Markdown references rewritten to
match. Raises subprocess.TimeoutExpired like convert_to_markdown().
`keep_mineru_output`, like in convert_to_markdown(), keeps mineru's raw
per-page output instead of discarding it — each page lands in its own
`<md stem>_mineru/<image stem>/` subdirectory, mirroring how the
extracted figures are namespaced per page.
A note spanning several pages OCRs each page with a separate `mineru`
invocation; left alone, that means reloading mineru's layout/OCR models
once per page. When MINERU_API_URL isn't already set (i.e. no caller is
already running a batch server across multiple notes) and there's more
than one page, this starts a MineruServer for the duration of the loop
so all of this note's pages share one already-warm server instead of
each page starting and stopping its own."""
_require_mineru()
image_paths = [Path(p) for p in image_paths]
if not image_paths:
raise ValueError("convert_images_to_markdown requires at least one image path")
images_dirname = f"{md_path.stem}_images"
parts = []
needs_server = len(image_paths) > 1 and MINERU_API_URL_ENV_VAR not in os.environ
server = MineruServer() if needs_server else contextlib.nullcontext()
with server, _mineru_out_dir(md_path, keep_mineru_output) as tmp_out:
for image_path in image_paths:
md_text, images_src = _run_mineru(image_path, tmp_out, timeout)
if images_src is not None:
page_dest = md_path.parent / images_dirname / image_path.stem
shutil.copytree(images_src, page_dest, dirs_exist_ok=True)
md_text = md_text.replace(
"](images/", f"]({images_dirname}/{image_path.stem}/"
)
parts.append(md_text.strip())
md_path.write_text(
html_tables_to_markdown("\n\n".join(parts)) + "\n", encoding="utf-8"
)