Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions packages/markitdown/src/markitdown/converters/_epub_converter.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import os
import posixpath
import zipfile
from urllib.parse import unquote
from defusedxml import minidom
from xml.dom.minidom import Document

from typing import BinaryIO, Any, Dict, List
from typing import BinaryIO, Any, Dict, List, Set

from ._html_converter import HtmlConverter
from .._base_converter import DocumentConverterResult
Expand Down Expand Up @@ -91,8 +93,9 @@ def convert(
base_path = "/".join(
opf_path.split("/")[:-1]
) # Get base directory of content.opf
zip_names = set(z.namelist())
spine = [
f"{base_path}/{manifest[item_id]}" if base_path else manifest[item_id]
self._resolve_manifest_href(manifest[item_id], base_path, zip_names)
for item_id in spine_order
if item_id in manifest
]
Expand Down Expand Up @@ -130,6 +133,29 @@ def convert(
markdown="\n\n".join(markdown_content), title=metadata["title"]
)

def _resolve_manifest_href(
self, href: str, base_path: str, zip_names: Set[str]
) -> str:
"""Resolve a manifest href to the matching ZIP entry name.

Manifest hrefs are URI references relative to the OPF, so reserved
characters such as spaces arrive percent-encoded, while ZIP entry names
are not encoded. Prefer the decoded form, but fall back to the raw href
so archives that store a literally-encoded name still resolve.
"""
candidates: List[str] = []
for candidate in (unquote(href), href):
resolved = posixpath.join(base_path, candidate) if base_path else candidate
resolved = posixpath.normpath(resolved)
if resolved not in candidates:
candidates.append(resolved)

for candidate in candidates:
if candidate in zip_names:
return candidate

return candidates[0]

def _get_text_from_node(self, dom: Document, tag_name: str) -> str | None:
"""Convenience function to extract a single occurrence of a tag (e.g., title)."""
texts = self._get_all_texts_from_nodes(dom, tag_name)
Expand Down
115 changes: 115 additions & 0 deletions packages/markitdown/tests/test_epub_converter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import io
import zipfile

from markitdown import StreamInfo
from markitdown.converters import EpubConverter

CONTAINER_XML = """<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
"""

CHAPTER_XHTML = """<html xmlns="http://www.w3.org/1999/xhtml">
<body><h1>{title}</h1><p>{body}</p></body>
</html>
"""


def _build_epub(manifest_items, spine_ids, documents) -> io.BytesIO:
"""Assemble a minimal EPUB from manifest entries and ZIP member names."""
manifest = "\n".join(
f'<item id="{item_id}" href="{href}" media-type="application/xhtml+xml"/>'
for item_id, href in manifest_items
)
spine = "\n".join(f'<itemref idref="{item_id}"/>' for item_id in spine_ids)
opf = f"""<?xml version="1.0"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="id">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:title>Encoded Hrefs</dc:title>
</metadata>
<manifest>{manifest}</manifest>
<spine>{spine}</spine>
</package>
"""

buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as z:
z.writestr("mimetype", "application/epub+zip")
z.writestr("META-INF/container.xml", CONTAINER_XML)
z.writestr("OEBPS/content.opf", opf)
for name, (title, body) in documents.items():
z.writestr(name, CHAPTER_XHTML.format(title=title, body=body))
buffer.seek(0)
return buffer


def _convert(stream: io.BytesIO) -> str:
result = EpubConverter().convert(
stream, StreamInfo(mimetype="application/epub+zip", extension=".epub")
)
# markdownify escapes underscores, so compare against unescaped text
return result.markdown.replace("\\", "")


def test_percent_encoded_href_resolves_to_zip_entry() -> None:
"""A space in a filename arrives percent-encoded in the manifest href."""
stream = _build_epub(
manifest_items=[("c1", "chapter%201.xhtml"), ("c2", "plain.xhtml")],
spine_ids=["c1", "c2"],
documents={
"OEBPS/chapter 1.xhtml": ("First", "SPACED_BODY"),
"OEBPS/plain.xhtml": ("Second", "PLAIN_BODY"),
},
)

markdown = _convert(stream)

assert "SPACED_BODY" in markdown, "percent-encoded href must resolve to its entry"
assert "PLAIN_BODY" in markdown, "unencoded hrefs must keep working"
assert markdown.index("SPACED_BODY") < markdown.index(
"PLAIN_BODY"
), "spine order is preserved"


def test_non_ascii_percent_encoded_href_resolves() -> None:
"""Non-ASCII filenames are percent-encoded UTF-8 in the manifest href."""
stream = _build_epub(
manifest_items=[("c1", "cap%C3%ADtulo.xhtml")],
spine_ids=["c1"],
documents={"OEBPS/capítulo.xhtml": ("Capítulo", "ACCENTED_BODY")},
)

assert "ACCENTED_BODY" in _convert(stream)


def test_literally_encoded_zip_entry_still_resolves() -> None:
"""An archive storing the encoded name verbatim keeps working."""
stream = _build_epub(
manifest_items=[("c1", "chapter%201.xhtml")],
spine_ids=["c1"],
documents={"OEBPS/chapter%201.xhtml": ("Literal", "LITERAL_BODY")},
)

assert "LITERAL_BODY" in _convert(stream)


def test_parent_relative_href_resolves() -> None:
"""Hrefs may point outside the OPF's own directory."""
stream = _build_epub(
manifest_items=[("c1", "../shared/chapter.xhtml")],
spine_ids=["c1"],
documents={"shared/chapter.xhtml": ("Shared", "SHARED_BODY")},
)

assert "SHARED_BODY" in _convert(stream)


if __name__ == "__main__":
test_percent_encoded_href_resolves_to_zip_entry()
test_non_ascii_percent_encoded_href_resolves()
test_literally_encoded_zip_entry_still_resolves()
test_parent_relative_href_resolves()
print("All tests passed")