diff --git a/docxtpl/subdoc.py b/docxtpl/subdoc.py index e8da093..1137c5e 100644 --- a/docxtpl/subdoc.py +++ b/docxtpl/subdoc.py @@ -16,6 +16,11 @@ import re +W14_ANCHOR_ID = ( + "{http://schemas.microsoft.com/office/word/2010/wordml}anchorId" +) + + class SubdocComposer(Composer): def attach_parts(self, doc, remove_property_fields=True): """Attach docx parts instead of appending the whole document @@ -29,6 +34,7 @@ def attach_parts(self, doc, remove_property_fields=True): cprops.dissolve_fields(name) self._create_style_id_mapping(doc) + self.renumber_ole_objects(doc) for element in doc.element.body: if isinstance(element, CT_SectPr): @@ -49,6 +55,71 @@ def attach_parts(self, doc, remove_property_fields=True): self.renumber_nvpicpr_ids() self.fix_section_types(doc) + def renumber_ole_objects(self, doc): + state = getattr(self.doc.part, "_docxtpl_ole_id_state", None) + if state is None: + body = self.doc.element.body + state = { + "shape_ids": set(xpath(body, ".//v:shape/@id")), + "shape_index": 1025, + "object_ids": set(xpath(body, ".//o:OLEObject/@ObjectID")), + "object_index": 1, + "anchor_ids": { + element.get(W14_ANCHOR_ID).upper() + for element in body.iter() + if element.get(W14_ANCHOR_ID) is not None + }, + "anchor_index": 1, + } + self.doc.part._docxtpl_ole_id_state = state + + state["shape_ids"].update( + xpath(doc.element.body, ".//v:shape/@id") + ) + for ole_object in xpath(doc.element.body, ".//o:OLEObject"): + old_shape_id = ole_object.get("ShapeID") + matching_shapes = [ + shape + for shape in xpath(ole_object.getparent(), "./v:shape") + if ( + old_shape_id is not None + and shape.get("id") == old_shape_id + ) + ] + if len(matching_shapes) == 1: + shape_id = self._next_ole_id( + state, "shape", lambda index: "_x0000_i%d" % index + ) + matching_shapes[0].set("id", shape_id) + ole_object.set("ShapeID", shape_id) + if ole_object.get("ObjectID") is not None: + ole_object.set( + "ObjectID", + self._next_ole_id( + state, "object", lambda index: "_%d" % index + ), + ) + + for element in doc.element.body.iter(): + if element.get(W14_ANCHOR_ID) is not None: + element.set( + W14_ANCHOR_ID, + self._next_ole_id( + state, "anchor", lambda index: "%08X" % index + ), + ) + + @staticmethod + def _next_ole_id(state, kind, formatter): + index_key = "%s_index" % kind + ids_key = "%s_ids" % kind + while formatter(state[index_key]) in state[ids_key]: + state[index_key] += 1 + value = formatter(state[index_key]) + state[index_key] += 1 + state[ids_key].add(value) + return value + def add_diagrams(self, doc, element): # While waiting docxcompose 1.3.3 dgm_rels = xpath(element, ".//dgm:relIds[@r:dm]") diff --git a/tests/_issue_621_ole.py b/tests/_issue_621_ole.py new file mode 100644 index 0000000..0d0a23f --- /dev/null +++ b/tests/_issue_621_ole.py @@ -0,0 +1,324 @@ +# -*- coding: utf-8 -*- + +from hashlib import sha256 +from io import BytesIO +import json +from pathlib import PurePosixPath +from zipfile import ZipFile + +from docx import Document +from lxml import etree + + +NAMESPACES = { + "o": "urn:schemas-microsoft-com:office:office", + "pr": "http://schemas.openxmlformats.org/package/2006/relationships", + "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main", + "v": "urn:schemas-microsoft-com:vml", + "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main", + "w14": "http://schemas.microsoft.com/office/word/2010/wordml", +} +PACKAGE_RELATIONSHIP = ( + "http://schemas.openxmlformats.org/officeDocument/2006/" + "relationships/package" +) +W14_ANCHOR_ID = "{%s}anchorId" % NAMESPACES["w14"] + + +def assert_file_identity(path, expected_size, expected_sha256): + data = path.read_bytes() + assert len(data) == expected_size, ( + "unexpected fixture size: %s" % path.name + ) + assert sha256(data).hexdigest() == expected_sha256, ( + "unexpected fixture SHA-256: %s" % path.name + ) + + +def _inspect_ooxml_package(data, required_main_part=None): + with ZipFile(BytesIO(data)) as package: + assert package.testzip() is None, ( + "embedded OOXML package has corrupt data" + ) + names = set(package.namelist()) + assert "[Content_Types].xml" in names, ( + "embedded target has no content types" + ) + etree.fromstring(package.read("[Content_Types].xml")) + + if required_main_part is None: + main_parts = sorted( + names + & { + "ppt/presentation.xml", + "word/document.xml", + "xl/workbook.xml", + } + ) + assert len(main_parts) == 1, ( + "embedded package has no unique OOXML main part" + ) + main_part = main_parts[0] + else: + assert required_main_part in names, ( + "embedded package lacks %s" % required_main_part + ) + main_part = required_main_part + + main_xml = etree.fromstring(package.read(main_part)) + sheet_names = ( + main_xml.xpath(".//s:sheet/@name", namespaces=NAMESPACES) + if main_part == "xl/workbook.xml" + else [] + ) + return main_part, sheet_names + + +def assert_xlsx_fixture(path, expected_size, expected_sha256, expected_sheet): + assert_file_identity(path, expected_size, expected_sha256) + main_part, sheet_names = _inspect_ooxml_package( + path.read_bytes(), required_main_part="xl/workbook.xml" + ) + assert main_part == "xl/workbook.xml" + assert sheet_names == [expected_sheet], ( + "unexpected workbook sheets in %s: %r" % (path.name, sheet_names) + ) + + +def add_ole_paragraph_anchor_ids(source, target): + anchor_count = 0 + with ZipFile(source) as source_archive, ZipFile( + target, "w" + ) as target_archive: + for item in source_archive.infolist(): + data = source_archive.read(item.filename) + if item.filename == "word/document.xml": + document_xml = etree.fromstring(data) + paragraphs = document_xml.xpath( + ".//w:p[.//o:OLEObject]", namespaces=NAMESPACES + ) + anchor_count = len(paragraphs) + for index, paragraph in enumerate(paragraphs, start=1): + paragraph.set(W14_ANCHOR_ID, "%08X" % index) + data = etree.tostring( + document_xml, + xml_declaration=True, + encoding="UTF-8", + standalone=True, + ) + target_archive.writestr(item, data) + return anchor_count + + +def add_non_ole_vml_shape(source, target, shape_id): + with ZipFile(source) as source_archive, ZipFile( + target, "w" + ) as target_archive: + for item in source_archive.infolist(): + data = source_archive.read(item.filename) + if item.filename == "word/document.xml": + document_xml = etree.fromstring(data) + ole_objects = document_xml.xpath( + ".//o:OLEObject", namespaces=NAMESPACES + ) + for index, ole_object in enumerate( + ole_objects, start=2048 + ): + old_shape_id = ole_object.get("ShapeID") + matching_shapes = ole_object.getparent().xpath( + "./v:shape[@id=$shape_id]", + namespaces=NAMESPACES, + shape_id=old_shape_id, + ) + assert len(matching_shapes) == 1 + new_shape_id = "_x0000_i%d" % index + matching_shapes[0].set("id", new_shape_id) + ole_object.set("ShapeID", new_shape_id) + body = document_xml.xpath( + ".//w:body", namespaces=NAMESPACES + )[0] + paragraph = etree.Element("{%s}p" % NAMESPACES["w"]) + run = etree.SubElement( + paragraph, "{%s}r" % NAMESPACES["w"] + ) + pict = etree.SubElement( + run, "{%s}pict" % NAMESPACES["w"] + ) + shape = etree.SubElement( + pict, "{%s}shape" % NAMESPACES["v"] + ) + shape.set("id", shape_id) + section = body.find("{%s}sectPr" % NAMESPACES["w"]) + if section is None: + body.append(paragraph) + else: + body.insert(body.index(section), paragraph) + data = etree.tostring( + document_xml, + xml_declaration=True, + encoding="UTF-8", + standalone=True, + ) + target_archive.writestr(item, data) + + +def _assert_unique(values, label, normalizer=None): + normalized = ( + [normalizer(value) for value in values] + if normalizer + else values + ) + assert len(set(normalized)) == len(normalized), "duplicate %s: %r" % ( + label, + values, + ) + + +def _embedding_target(relationship): + target = relationship.get("Target") + assert target, "OLE package relationship has no target" + relative = PurePosixPath(target) + assert not relative.is_absolute(), ( + "external OLE package target: %s" % target + ) + assert ".." not in relative.parts, ( + "traversing OLE package target: %s" % target + ) + package_path = PurePosixPath("word") / relative + assert package_path.parts[:2] == ("word", "embeddings"), ( + "OLE target is outside word/embeddings: %s" % target + ) + return package_path.as_posix() + + +def assert_ole_integrity( + result, + expected_count, + expected_anchor_count, + required_embedding_member=None, + expected_embedding_sha256=None, + expected_workbook_sheets=None, +): + Document(result) + with ZipFile(result) as archive: + assert archive.testzip() is None, "result DOCX has corrupt ZIP data" + document_xml = etree.fromstring(archive.read("word/document.xml")) + relationships_xml = etree.fromstring( + archive.read("word/_rels/document.xml.rels") + ) + archive_names = set(archive.namelist()) + relationships = { + relationship.get("Id"): relationship + for relationship in relationships_xml.xpath( + ".//pr:Relationship", namespaces=NAMESPACES + ) + } + ole_objects = document_xml.xpath( + ".//o:OLEObject", namespaces=NAMESPACES + ) + assert len(ole_objects) == expected_count, ( + "expected %d OLE objects, found %d" + % (expected_count, len(ole_objects)) + ) + + shape_ids = [] + object_ids = [] + relationship_ids = [] + embedding_targets = [] + embedding_sha256 = [] + embedded_main_parts = [] + workbook_sheet_names = [] + for ole_object in ole_objects: + shape_id = ole_object.get("ShapeID") + object_id = ole_object.get("ObjectID") + relationship_id = ole_object.get("{%s}id" % NAMESPACES["r"]) + assert shape_id, "OLE object has no ShapeID" + assert object_id, "OLE object has no ObjectID" + assert relationship_id, "OLE object has no relationship ID" + + matching_shapes = ole_object.getparent().xpath( + "./v:shape[@id=$shape_id]", + namespaces=NAMESPACES, + shape_id=shape_id, + ) + assert len(matching_shapes) == 1, ( + "OLE ShapeID does not resolve to one sibling shape: %s" + % shape_id + ) + + relationship = relationships.get(relationship_id) + assert relationship is not None, ( + "missing OLE relationship: %s" % relationship_id + ) + assert relationship.get("TargetMode") is None, ( + "OLE package relationship must be internal: %s" + % relationship_id + ) + assert relationship.get("Type") == PACKAGE_RELATIONSHIP, ( + "unexpected OLE relationship type: %s" % relationship_id + ) + embedding_target = _embedding_target(relationship) + assert embedding_target in archive_names, ( + "missing embedded package part: %s" % embedding_target + ) + + embedded_data = archive.read(embedding_target) + main_part, sheet_names = _inspect_ooxml_package( + embedded_data, required_embedding_member + ) + shape_ids.append(shape_id) + object_ids.append(object_id) + relationship_ids.append(relationship_id) + embedding_targets.append(embedding_target) + embedding_sha256.append(sha256(embedded_data).hexdigest()) + embedded_main_parts.append(main_part) + workbook_sheet_names.extend(sheet_names) + + assert len(shape_ids) == expected_count + assert len(object_ids) == expected_count + assert len(relationship_ids) == expected_count + assert len(embedding_targets) == expected_count + _assert_unique(shape_ids, "OLE ShapeID values") + _assert_unique(object_ids, "OLE ObjectID values") + _assert_unique(relationship_ids, "OLE relationship IDs") + _assert_unique(embedding_targets, "OLE embedding targets") + + all_shape_ids = document_xml.xpath(".//v:shape/@id", namespaces=NAMESPACES) + assert all(all_shape_ids), "empty VML shape ID" + _assert_unique(all_shape_ids, "VML shape ID values") + + anchor_ids = document_xml.xpath(".//@w14:anchorId", namespaces=NAMESPACES) + assert len(anchor_ids) == expected_anchor_count, ( + "expected %d w14:anchorId values, found %d" + % (expected_anchor_count, len(anchor_ids)) + ) + assert all(anchor_ids), "empty w14:anchorId value" + _assert_unique( + anchor_ids, + "w14:anchorId values after hexadecimal normalization", + normalizer=lambda value: value.upper(), + ) + + if expected_embedding_sha256 is not None: + assert embedding_sha256 == list(expected_embedding_sha256), ( + "embedded payload SHA-256 values changed: %r" % embedding_sha256 + ) + if expected_workbook_sheets is not None: + assert workbook_sheet_names == list(expected_workbook_sheets), ( + "embedded workbook sheets changed: %r" % workbook_sheet_names + ) + + summary = { + "anchor_count": len(anchor_ids), + "embedded_main_parts": sorted(embedded_main_parts), + "embedding_sha256": sorted(embedding_sha256), + "embedding_target_count": len(embedding_targets), + "object_id_count": len(object_ids), + "ole_count": len(ole_objects), + "relationship_id_count": len(relationship_ids), + "shape_id_count": len(shape_ids), + "workbook_sheet_names": sorted(workbook_sheet_names), + } + print(json.dumps(summary, sort_keys=True)) + return summary diff --git a/tests/embedded_ole_subdoc.py b/tests/embedded_ole_subdoc.py new file mode 100644 index 0000000..8e70eea --- /dev/null +++ b/tests/embedded_ole_subdoc.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +from pathlib import Path +from tempfile import TemporaryDirectory + +from docx import Document +from docxtpl import DocxTemplate + +from _issue_621_ole import ( + W14_ANCHOR_ID, + add_non_ole_vml_shape, + add_ole_paragraph_anchor_ids, + assert_file_identity, + assert_ole_integrity, +) + + +SOURCE_FIXTURE = Path("templates/embedded_main_tpl.docx") +SOURCE_FIXTURE_SHA256 = ( + "40fa08a534110d411da00f38ed5ba971c2dcd12849e95edb3a3a9a48ca29919f" +) + + +assert_file_identity(SOURCE_FIXTURE, 164480, SOURCE_FIXTURE_SHA256) +with TemporaryDirectory() as temporary_directory: + output_dir = Path(temporary_directory) + anchored_source = output_dir / "anchored_source.docx" + main_template = output_dir / "main_with_existing_ole.docx" + result = output_dir / "embedded_ole_subdoc.docx" + mixed_source = output_dir / "mixed_vml_and_ole.docx" + mixed_main = output_dir / "mixed_vml_main.docx" + mixed_result = output_dir / "mixed_vml_result.docx" + + source_anchor_count = add_ole_paragraph_anchor_ids( + SOURCE_FIXTURE, anchored_source + ) + assert source_anchor_count == 3 + main_document = Document(anchored_source) + for index in range(source_anchor_count + 1, 11): + paragraph = main_document.add_paragraph("reserved anchor %d" % index) + anchor_id = "%08X" % index + paragraph._p.set( + W14_ANCHOR_ID, + anchor_id.lower() if index == 10 else anchor_id, + ) + main_document.add_paragraph("{{p first_subdoc }}") + main_document.add_paragraph("{{p second_subdoc }}") + main_document.save(main_template) + + template = DocxTemplate(main_template) + template.render( + { + "first_subdoc": template.new_subdoc(anchored_source), + "second_subdoc": template.new_subdoc(anchored_source), + } + ) + template.save(result) + + summary = assert_ole_integrity( + result, expected_count=12, expected_anchor_count=16 + ) + assert summary["ole_count"] == 4 + (2 * 4) + + add_non_ole_vml_shape( + anchored_source, mixed_source, "_x0000_i1025" + ) + mixed_main_document = Document() + mixed_main_document.add_paragraph("{{p mixed_subdoc }}") + mixed_main_document.save(mixed_main) + mixed_template = DocxTemplate(mixed_main) + mixed_template.render( + {"mixed_subdoc": mixed_template.new_subdoc(mixed_source)} + ) + mixed_template.save(mixed_result) + mixed_summary = assert_ole_integrity( + mixed_result, expected_count=4, expected_anchor_count=3 + ) + assert mixed_summary["ole_count"] == 4 diff --git a/tests/issue_621_public_reproduction.py b/tests/issue_621_public_reproduction.py new file mode 100644 index 0000000..50fdd0e --- /dev/null +++ b/tests/issue_621_public_reproduction.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +from pathlib import Path +from tempfile import TemporaryDirectory + +from docxtpl import DocxTemplate + +from _issue_621_ole import ( + assert_file_identity, + assert_ole_integrity, + assert_xlsx_fixture, +) + + +MAIN_FIXTURE = Path("templates/issue_621_main.docx") +SUBDOC_FIXTURE = Path("templates/issue_621_subdoc.docx") +EMBEDDING_ZIPNAME = "word/embeddings/Microsoft_Excel_Worksheet.xlsx" +XLSX_FIXTURES = [ + ( + Path("templates/issue_621_excel_0.xlsx"), + 4887, + "3b7d1befbe9fe6d4dfa8bee06b3dc424774b37079a884e89427c78577516805f", + "No.1", + ), + ( + Path("templates/issue_621_excel_1.xlsx"), + 4867, + "d353a37445081eb1dd471a5f9cdeec55a5222de128483d8998283776c6186f15", + "No.2", + ), + ( + Path("templates/issue_621_excel_2.xlsx"), + 4868, + "291cf1832083f0c0476679fb0030c1f818c079ef4326e5e064b9ce76c541124e", + "No.3", + ), + ( + Path("templates/issue_621_excel_3.xlsx"), + 4867, + "ff759e011e2cf1216aea110819e46cde04f3726a6c7b6d0b67b53cf25a7535a3", + "No.4", + ), + ( + Path("templates/issue_621_excel_4.xlsx"), + 4867, + "7a26b676c693c8f58deeec4879751257a02d6955957a0fcad5bbdef9ada55396", + "No.5", + ), + ( + Path("templates/issue_621_excel_5.xlsx"), + 4867, + "a4f3b5c2de16078dbb77757437fbbeca1ceb2e7d99d99b2dbaba609cf9b83197", + "No.6", + ), + ( + Path("templates/issue_621_excel_6.xlsx"), + 4868, + "51d03cae946ff4763a29049c08acd6a2af5cb1fc167fbce25d5ca53c96227ed6", + "No.7", + ), +] + + +assert_file_identity( + MAIN_FIXTURE, + 12772, + "8445e28450aa911b92d83b251ff634c2f49d8216a6cc829832965ef984f14ca0", +) +assert_file_identity( + SUBDOC_FIXTURE, + 23798, + "c57714a23c1d17f991f95dff21c1e6931195fd7f8848258e82a34586bb5a302d", +) +for fixture in XLSX_FIXTURES: + assert_xlsx_fixture(*fixture) + +with TemporaryDirectory() as temporary_directory: + output_dir = Path(temporary_directory) + result = output_dir / "issue_621_public_reproduction.docx" + template = DocxTemplate(MAIN_FIXTURE) + sub_docs = [] + for index, (xlsx_path, _, _, _) in enumerate(XLSX_FIXTURES): + subdoc_path = output_dir / ("issue_621_subdoc_%d.docx" % index) + subdoc_template = DocxTemplate(SUBDOC_FIXTURE) + subdoc_template.replace_zipname(EMBEDDING_ZIPNAME, str(xlsx_path)) + subdoc_template.save(subdoc_path) + sub_docs.append(template.new_subdoc(subdoc_path)) + + template.render({"sub_docs": sub_docs}) + template.save(result) + summary = assert_ole_integrity( + result, + expected_count=7, + expected_anchor_count=7, + required_embedding_member="xl/workbook.xml", + expected_embedding_sha256=[item[2] for item in XLSX_FIXTURES], + expected_workbook_sheets=[item[3] for item in XLSX_FIXTURES], + ) + assert summary["ole_count"] == len(XLSX_FIXTURES) diff --git a/tests/templates/issue_621_excel_0.xlsx b/tests/templates/issue_621_excel_0.xlsx new file mode 100644 index 0000000..5fd51b2 Binary files /dev/null and b/tests/templates/issue_621_excel_0.xlsx differ diff --git a/tests/templates/issue_621_excel_1.xlsx b/tests/templates/issue_621_excel_1.xlsx new file mode 100644 index 0000000..f51a28b Binary files /dev/null and b/tests/templates/issue_621_excel_1.xlsx differ diff --git a/tests/templates/issue_621_excel_2.xlsx b/tests/templates/issue_621_excel_2.xlsx new file mode 100644 index 0000000..5ffb617 Binary files /dev/null and b/tests/templates/issue_621_excel_2.xlsx differ diff --git a/tests/templates/issue_621_excel_3.xlsx b/tests/templates/issue_621_excel_3.xlsx new file mode 100644 index 0000000..1af1a5e Binary files /dev/null and b/tests/templates/issue_621_excel_3.xlsx differ diff --git a/tests/templates/issue_621_excel_4.xlsx b/tests/templates/issue_621_excel_4.xlsx new file mode 100644 index 0000000..cce5337 Binary files /dev/null and b/tests/templates/issue_621_excel_4.xlsx differ diff --git a/tests/templates/issue_621_excel_5.xlsx b/tests/templates/issue_621_excel_5.xlsx new file mode 100644 index 0000000..a74ab36 Binary files /dev/null and b/tests/templates/issue_621_excel_5.xlsx differ diff --git a/tests/templates/issue_621_excel_6.xlsx b/tests/templates/issue_621_excel_6.xlsx new file mode 100644 index 0000000..fd2c91c Binary files /dev/null and b/tests/templates/issue_621_excel_6.xlsx differ diff --git a/tests/templates/issue_621_main.docx b/tests/templates/issue_621_main.docx new file mode 100644 index 0000000..a859901 Binary files /dev/null and b/tests/templates/issue_621_main.docx differ diff --git a/tests/templates/issue_621_subdoc.docx b/tests/templates/issue_621_subdoc.docx new file mode 100644 index 0000000..e560d3d Binary files /dev/null and b/tests/templates/issue_621_subdoc.docx differ