From a75923b813f0a84894b4f72b085e8f1cf412cddb Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Fri, 23 Oct 2026 07:12:18 +0100 Subject: [PATCH 1/4] Tika ParserBolt: apply the JSoupFilters to the documents parsed by Tika A new JSoupBuilder turns the XHTML SAX events produced by Tika into a JSoup document, the same way DOMBuilder does for the W3C DOM given to the parse filters. It is teed with the link and text handlers, so the document is parsed once, and it also receives the content replayed from a Tika Pipes fork. The document is only built when JSoupFilters are configured (jsoup.filters.config.file, as for JSoupParserBolt); they run before the parse filters. Co-Authored-By: Claude Opus 5.5 --- .../stormcrawler/parse/JSoupFilters.java | 5 + external/tika/README.md | 6 +- .../stormcrawler/tika/JSoupBuilder.java | 114 ++++++++++++++++ .../apache/stormcrawler/tika/ParserBolt.java | 28 +++- .../stormcrawler/tika/JSoupBuilderTest.java | 84 ++++++++++++ .../tika/ParserBoltJSoupFiltersTest.java | 124 ++++++++++++++++++ .../test/resources/test.pdf.jsoupfilters.json | 11 ++ 7 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java create mode 100644 external/tika/src/test/java/org/apache/stormcrawler/tika/JSoupBuilderTest.java create mode 100644 external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java create mode 100644 external/tika/src/test/resources/test.pdf.jsoupfilters.json diff --git a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java index b69432d5e..62ee4e1b8 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java @@ -111,6 +111,11 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode filters = list.toArray(new JSoupFilter[0]); } + /** Returns true if no filters are configured, so callers can skip building a document. */ + public boolean isEmpty() { + return filters.length == 0; + } + @Override public void filter( @NotNull String url, diff --git a/external/tika/README.md b/external/tika/README.md index d342d740a..9340f92a2 100644 --- a/external/tika/README.md +++ b/external/tika/README.md @@ -45,6 +45,10 @@ The time spent parsing a document can be limited with `parser.tika.timeout` (mil Each `ParserBolt` instance runs one fork, so a host needs the worker heap plus one fork heap per `ParserBolt` instance on it, and some overhead for each JVM. When several forks share a host, consider `-XX:ActiveProcessorCount` as well as a heap limit. A handful of related keys tune the forked JVMs: `parser.tika.pipes.jvmargs` (`-Xmx512m` is added when they set no maximum heap), `parser.tika.pipes.maxfilesperprocess` (restart a fork after this many documents, to bound slow leaks in parsing libraries; Tika's default is 10000), and `parser.tika.pipes.plugins.dir` (Tika Pipes plugins, not needed by default). Documents over 10MB are handed to the forked JVM through a temporary file in the worker's `java.io.tmpdir`. -`parser.htmlmapper.classname` is not applied to parses running under `parser.tika.timeout`: an `HtmlMapper` cannot be passed to the forked JVM, which always uses Tika's `DefaultHtmlMapper`. That mapper drops the elements it does not consider safe from the DOM given to the parse filters, so XPath expressions written against the default `IdentityHtmlMapper` output may need adjusting. +## JSoup filters + +The JSoupFilters configured with `jsoup.filters.config.file` are applied to every document parsed by Tika, not only HTML, the same way as with the `JSoupParserBolt`; they run before the parse filters. The JSoup document is built from the XHTML Tika produces, not from the original content: a PDF becomes `
` and `

` elements, and the `` of an HTML page holds Tika's metadata (e.g. `` instead of ``), so expressions written for the original HTML may need adjusting. Element names are lowercased. The document is only built if at least one JSoupFilter is configured. + +`parser.htmlmapper.classname` is not applied to parses running under `parser.tika.timeout`: an `HtmlMapper` cannot be passed to the forked JVM, which always uses Tika's `DefaultHtmlMapper`. That mapper drops the elements it does not consider safe from the documents given to the JSoup and parse filters, so XPath expressions written against the default `IdentityHtmlMapper` output may need adjusting. Since Tika 4, Tika metadata keys use namespaced names, which surface as renamed `parse.*` keys, e.g. `parse.resourceName` is now `parse.tk:resource-name`. diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java new file mode 100644 index 000000000..11e90debd --- /dev/null +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import org.apache.commons.lang3.StringUtils; +import org.jsoup.nodes.DataNode; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.TextNode; +import org.jsoup.parser.ParseSettings; +import org.jsoup.parser.Tag; +import org.xml.sax.Attributes; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Builds a JSoup document from the XHTML SAX events produced by Tika, so that JSoupFilters can be + * applied to any format Tika can parse. Element names are lowercased, namespaces are dropped. The + * document holds whatever was received so far if the parse stops early, e.g. when the text limit is + * reached. + */ +public class JSoupBuilder extends DefaultHandler { + + private final Document document; + + private Element current; + + private final StringBuilder text = new StringBuilder(); + + /** + * @param baseUri URL of the document, used to resolve relative links + */ + public JSoupBuilder(String baseUri) { + document = new Document(baseUri); + current = document; + } + + public Document getDocument() { + flushText(); + return document; + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes atts) { + flushText(); + String name = StringUtils.isNotEmpty(localName) ? localName : qName; + Element element = + new Element(Tag.valueOf(name, ParseSettings.htmlDefault), document.baseUri()); + for (int i = 0; i < atts.getLength(); i++) { + String attName = atts.getLocalName(i); + if (StringUtils.isEmpty(attName)) { + attName = atts.getQName(i); + } + if (attName.startsWith("xmlns")) { + continue; + } + element.attr(attName, atts.getValue(i)); + } + current.appendChild(element); + current = element; + } + + @Override + public void endElement(String uri, String localName, String qName) { + flushText(); + if (current != document) { + current = current.parent(); + } + } + + @Override + public void characters(char[] ch, int start, int length) { + text.append(ch, start, length); + } + + @Override + public void ignorableWhitespace(char[] ch, int start, int length) { + text.append(ch, start, length); + } + + /** Adds the text received since the last element event as a single node. */ + private void flushText() { + if (text.length() == 0) { + return; + } + String value = text.toString(); + text.setLength(0); + // nothing outside of the root element + if (current == document) { + return; + } + // keep the content of scripts and styles as is, like the JSoup parser does + String name = current.normalName(); + if ("script".equals(name) || "style".equals(name)) { + current.appendChild(new DataNode(value)); + } else { + current.appendChild(new TextNode(value)); + } + } +} diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 3d355171c..2405955cf 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -58,6 +58,7 @@ import org.apache.stormcrawler.filtering.URLFilters; import org.apache.stormcrawler.metrics.CrawlerMetrics; import org.apache.stormcrawler.metrics.ScopedCounter; +import org.apache.stormcrawler.parse.JSoupFilters; import org.apache.stormcrawler.parse.Outlink; import org.apache.stormcrawler.parse.ParseData; import org.apache.stormcrawler.parse.ParseFilter; @@ -188,6 +189,7 @@ private static SAXParserFactory newHardenedSaxParserFactory() { private URLFilters urlFilters = null; private ParseFilter parseFilters = null; + private JSoupFilters jsoupFilters = null; private OutputCollector collector; @@ -239,6 +241,8 @@ public void prepare( parseFilters = ParseFilters.fromConf(conf); + jsoupFilters = JSoupFilters.fromConf(conf); + upperCaseElementNames = ConfUtils.getBoolean(conf, "parser.uppercase.element.names", true); extractEmbedded = ConfUtils.getBoolean(conf, "parser.extract.embedded", false); @@ -395,7 +399,7 @@ public void execute(Tuple tuple) { LinkContentHandler linkHandler = new LinkContentHandler(); ContentHandler textHandler = new BodyContentHandler(textMaxLength); - TeeContentHandler teeHandler = new TeeContentHandler(linkHandler, textHandler); + List handlers = new ArrayList<>(List.of(linkHandler, textHandler)); // seed the context with the components configured in the // "parse-context" section of the Tika configuration ParseContext parseContext = createParseContext(); @@ -424,9 +428,19 @@ public void execute(Tuple tuple) { DOMBuilder domhandler = new DOMBuilder(doc, root); domhandler.setUpperCaseElementNames(upperCaseElementNames); domhandler.setDefaultNamespaceURI(XHTMLContentHandler.XHTML); - teeHandler = new TeeContentHandler(linkHandler, textHandler, domhandler); + handlers.add(domhandler); + } + + // build a JSoup document if there are JSoupFilters + JSoupBuilder jsoupBuilder = null; + if (!jsoupFilters.isEmpty()) { + jsoupBuilder = new JSoupBuilder(url); + handlers.add(jsoupBuilder); } + TeeContentHandler teeHandler = + new TeeContentHandler(handlers.toArray(new ContentHandler[0])); + // parse String text; boolean textTrimmed = false; @@ -491,6 +505,16 @@ public void execute(Tuple tuple) { parseData.setText(text); parseData.setContent(content); + // apply the JSoup filters if any + if (jsoupBuilder != null) { + try { + jsoupFilters.filter(url, content, jsoupBuilder.getDocument(), parse); + } catch (RuntimeException e) { + handleException(url, e, metadata, tuple, "jsoup filtering"); + return; + } + } + // apply the parse filters if any try { parseFilters.filter(url, content, root, parse); diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/JSoupBuilderTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/JSoupBuilderTest.java new file mode 100644 index 000000000..37667ea1f --- /dev/null +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/JSoupBuilderTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import java.io.StringReader; +import javax.xml.parsers.SAXParserFactory; +import org.jsoup.nodes.DataNode; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; + +class JSoupBuilderTest { + + private static Document build(String xhtml) throws Exception { + SAXParserFactory factory = SAXParserFactory.newInstance(); + factory.setNamespaceAware(true); + XMLReader reader = factory.newSAXParser().getXMLReader(); + JSoupBuilder builder = new JSoupBuilder("https://example.com/dir/page.html"); + reader.setContentHandler(builder); + reader.parse(new InputSource(new StringReader(xhtml))); + return builder.getDocument(); + } + + @Test + void testStructure() throws Exception { + Document doc = + build( + "T" + + "" + + "

one & two three

" + + "link"); + + Assertions.assertEquals("T", doc.title()); + Assertions.assertNotNull(doc.body()); + + Element p = doc.selectFirst("p.x"); + Assertions.assertNotNull(p); + Assertions.assertEquals("one & two three", p.text()); + // text split by the parser into several events ends up in a single node + Assertions.assertEquals(3, p.childNodeSize()); + + Element script = doc.selectFirst("script"); + Assertions.assertInstanceOf(DataNode.class, script.childNode(0)); + Assertions.assertEquals("if (a < b) {}", script.data()); + + Element a = doc.selectFirst("a"); + Assertions.assertEquals("https://example.com/dir/other.html", a.absUrl("href")); + Assertions.assertFalse(doc.selectFirst("html").hasAttr("xmlns")); + } + + @Test + void testTruncatedInput() throws Exception { + SAXParserFactory factory = SAXParserFactory.newInstance(); + factory.setNamespaceAware(true); + XMLReader reader = factory.newSAXParser().getXMLReader(); + JSoupBuilder builder = new JSoupBuilder("https://example.com/"); + reader.setContentHandler(builder); + Assertions.assertThrows( + Exception.class, + () -> + reader.parse( + new InputSource( + new StringReader("

kept

cut")))); + Assertions.assertEquals("kept", builder.getDocument().selectFirst("p").text()); + } +} diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java new file mode 100644 index 000000000..0c92f9035 --- /dev/null +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.apache.storm.task.OutputCollector; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.TestUtil; +import org.apache.stormcrawler.parse.ParsingTester; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +/** Checks that the ParserBolt applies the JSoupFilters to the documents parsed by Tika. */ +class ParserBoltJSoupFiltersTest extends ParsingTester { + + private static byte[] pdf; + + @BeforeAll + static void createPdf() throws IOException { + try (PDDocument doc = new PDDocument(); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + PDPage page = new PDPage(); + doc.addPage(page); + try (PDPageContentStream content = new PDPageContentStream(doc, page)) { + content.beginText(); + content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + content.newLineAtOffset(100, 700); + content.showText("alpha"); + content.endText(); + } + doc.save(out); + pdf = out.toByteArray(); + } + } + + @BeforeEach + void setupParserBolt() { + bolt = new ParserBolt(); + setupParserBolt(bolt); + } + + private void prepare(String jsoupConfig, Map conf) { + conf.put("jsoup.filters.config.file", jsoupConfig); + bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + } + + private Metadata emittedMetadata() { + List> emitted = output.getEmitted(); + Assertions.assertEquals(1, emitted.size()); + return (Metadata) emitted.get(0).get(2); + } + + @Test + void testHtml() throws IOException { + // same expressions as the JSoupParserBolt tests in core; the meta elements are not + // checked as Tika rewrites the head from its own metadata (e.g. dc:subject) + prepare("test.xpath.jsoupfilters.json", new HashMap<>()); + parse("https://example.com", "xpath-test.html"); + Metadata metadata = emittedMetadata(); + Assertions.assertEquals("XPath Test Page", metadata.getFirstValue("title_text")); + Assertions.assertEquals( + "Hello world, this is a test.", metadata.getFirstValue("tidy_text").strip()); + Assertions.assertTrue(metadata.getFirstValue("html_content").contains("world")); + Assertions.assertEquals(2, metadata.getValues("concept_upper").length); + Assertions.assertTrue(metadata.getFirstValue("fallback").contains("Main Title")); + } + + @Test + void testPdf() throws IOException { + prepare("test.pdf.jsoupfilters.json", new HashMap<>()); + parse("https://example.org/doc.pdf", pdf, new Metadata()); + Assertions.assertEquals("alpha", emittedMetadata().getFirstValue("page_text").strip()); + } + + @Test + @Timeout(60) + void testPdfWithPipes() throws IOException { + Map conf = new HashMap<>(); + conf.put(ParserBolt.PARSE_TIMEOUT_PARAM, 30_000L); + conf.put(ParserBolt.PIPES_JVM_ARGS_PARAM, "-Xmx256m"); + prepare("test.pdf.jsoupfilters.json", conf); + try { + parse("https://example.org/doc.pdf", pdf, new Metadata()); + Assertions.assertEquals("alpha", emittedMetadata().getFirstValue("page_text").strip()); + } finally { + bolt.cleanup(); + } + } + + @Test + void testNoJSoupFilters() throws IOException { + bolt.prepare( + new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + parse("https://example.org/doc.pdf", pdf, new Metadata()); + Assertions.assertNull(emittedMetadata().getFirstValue("page_text")); + } +} diff --git a/external/tika/src/test/resources/test.pdf.jsoupfilters.json b/external/tika/src/test/resources/test.pdf.jsoupfilters.json new file mode 100644 index 000000000..298d9db5f --- /dev/null +++ b/external/tika/src/test/resources/test.pdf.jsoupfilters.json @@ -0,0 +1,11 @@ +{ + "org.apache.stormcrawler.parse.JSoupFilters": [ + { + "class": "org.apache.stormcrawler.jsoup.XPathFilter", + "name": "XPathFilter", + "params": { + "page_text": "//DIV[@class=\"page\"]" + } + } + ] +} From 2d23e89f03bfdbf8a51992a8a8700c1a8dc2441c Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Fri, 23 Oct 2026 07:17:19 +0100 Subject: [PATCH 2/4] Tika ParserBolt: use the IdentityHtmlMapper in the Tika Pipes fork Tika only sends the ParseContext components listed in its META-INF/tika/*.idx files to the forked JVM, which used its DefaultHtmlMapper as a result and dropped elements such as span and most attributes from the documents given to the filters. Register IdentityHtmlMapper in a parse-context.idx shipped by this module and pass it with each parse when it is the configured mapper (the default). Co-Authored-By: Claude Opus 5.5 --- external/tika/README.md | 2 +- .../apache/stormcrawler/tika/ParserBolt.java | 20 +++++++++++--- .../resources/META-INF/tika/parse-context.idx | 20 ++++++++++++++ .../tika/ParserBoltJSoupFiltersTest.java | 27 ++++++++++++++++--- 4 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 external/tika/src/main/resources/META-INF/tika/parse-context.idx diff --git a/external/tika/README.md b/external/tika/README.md index 9340f92a2..728c371b7 100644 --- a/external/tika/README.md +++ b/external/tika/README.md @@ -49,6 +49,6 @@ Each `ParserBolt` instance runs one fork, so a host needs the worker heap plus o The JSoupFilters configured with `jsoup.filters.config.file` are applied to every document parsed by Tika, not only HTML, the same way as with the `JSoupParserBolt`; they run before the parse filters. The JSoup document is built from the XHTML Tika produces, not from the original content: a PDF becomes `

` and `

` elements, and the `` of an HTML page holds Tika's metadata (e.g. `` instead of ``), so expressions written for the original HTML may need adjusting. Element names are lowercased. The document is only built if at least one JSoupFilter is configured. -`parser.htmlmapper.classname` is not applied to parses running under `parser.tika.timeout`: an `HtmlMapper` cannot be passed to the forked JVM, which always uses Tika's `DefaultHtmlMapper`. That mapper drops the elements it does not consider safe from the documents given to the JSoup and parse filters, so XPath expressions written against the default `IdentityHtmlMapper` output may need adjusting. +Under `parser.tika.timeout`, the default `IdentityHtmlMapper` is passed to the forked JVM, so the JSoup and parse filters get the same documents as with an in-process parse. Tika can only send the components registered in a `META-INF/tika/*.idx` file to the forked JVM: this module registers `IdentityHtmlMapper` in its `META-INF/tika/parse-context.idx`, which must be merged with the ones from Tika when building a shaded jar (see the `AppendingTransformer` entries in the archetype's `pom.xml`). Any other value of `parser.htmlmapper.classname` is ignored in this mode and the forked JVM uses Tika's `DefaultHtmlMapper`, which drops the elements it does not consider safe. Since Tika 4, Tika metadata keys use namespaced names, which surface as renamed `parse.*` keys, e.g. `parse.resourceName` is now `parse.tk:resource-name`. diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 2405955cf..8bc3508cf 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -223,6 +223,12 @@ private static SAXParserFactory newHardenedSaxParserFactory() { /** Runs parses under {@link #PARSE_TIMEOUT_PARAM} in a forked JVM; null otherwise. */ private PipesForkParser pipesForkParser; + /** + * HtmlMapper sent to the forked JVM, which uses Tika's DefaultHtmlMapper when null. Only the + * IdentityHtmlMapper can be sent, as it is registered in META-INF/tika/parse-context.idx. + */ + private HtmlMapper pipesHtmlMapper; + /** On-disk copy of the Tika configuration, kept alive to merge into the forked JVM's config. */ private Path resolvedTikaConfigPath; @@ -638,12 +644,15 @@ private PipesForkParser buildPipesForkParser(Map conf) { } pipesConfig.setJvmArgs(forkedJvmArgs(conf)); - // an HtmlMapper can't be passed to the forked JVM, which always uses Tika's default - if (!DefaultHtmlMapper.class.equals(htmlMapperClass)) { + // Tika only sends the components registered in its index files to the forked JVM, + // which falls back to its default mapper otherwise + if (IdentityHtmlMapper.class.equals(htmlMapperClass)) { + pipesHtmlMapper = IdentityHtmlMapper.INSTANCE; + } else if (!DefaultHtmlMapper.class.equals(htmlMapperClass)) { LOG.warn( "parser.htmlmapper.classname ({}) is ignored under {}: the forked JVM maps" - + " HTML with {}, which drops some elements from the DOM given to" - + " the parse filters", + + " HTML with {}, which drops some elements from the documents given" + + " to the JSoup and parse filters", htmlMapperClass.getName(), PARSE_TIMEOUT_PARAM, DefaultHtmlMapper.class.getSimpleName()); @@ -755,6 +764,9 @@ private PipesParseOutcome parseWithPipes( // the fork loads the "parse-context" section of the Tika configuration itself ParseContext parseContext = new ParseContext(); setPdfParserConfig(parseContext); + if (pipesHtmlMapper != null) { + parseContext.set(HtmlMapper.class, pipesHtmlMapper); + } result = pipesForkParser.parse(tis, seedMetadata, parseContext); } catch (TikaException | PipesException e) { throw new ParsePipesInfraException("Tika Pipes error for " + url, e); diff --git a/external/tika/src/main/resources/META-INF/tika/parse-context.idx b/external/tika/src/main/resources/META-INF/tika/parse-context.idx new file mode 100644 index 000000000..91c652220 --- /dev/null +++ b/external/tika/src/main/resources/META-INF/tika/parse-context.idx @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Registers Tika's IdentityHtmlMapper so that the ParserBolt can send it to +# the forked JVMs used by Tika Pipes (parser.tika.timeout); Tika only +# serialises the ParseContext components which are listed in these files. +# Format: component-name=fully.qualified.ClassName[:key=contextKeyClass] +identity-html-mapper=org.apache.tika.parser.html.IdentityHtmlMapper:key=org.apache.tika.parser.html.HtmlMapper diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java index 0c92f9035..0a18456cd 100644 --- a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltJSoupFiltersTest.java @@ -77,21 +77,40 @@ private Metadata emittedMetadata() { return (Metadata) emitted.get(0).get(2); } - @Test - void testHtml() throws IOException { + private void checkHtml(Map conf) throws IOException { // same expressions as the JSoupParserBolt tests in core; the meta elements are not // checked as Tika rewrites the head from its own metadata (e.g. dc:subject) - prepare("test.xpath.jsoupfilters.json", new HashMap<>()); - parse("https://example.com", "xpath-test.html"); + prepare("test.xpath.jsoupfilters.json", conf); + try { + parse("https://example.com", "xpath-test.html"); + } finally { + bolt.cleanup(); + } Metadata metadata = emittedMetadata(); Assertions.assertEquals("XPath Test Page", metadata.getFirstValue("title_text")); Assertions.assertEquals( "Hello world, this is a test.", metadata.getFirstValue("tidy_text").strip()); Assertions.assertTrue(metadata.getFirstValue("html_content").contains("world")); + // span elements and class attributes are dropped by Tika's DefaultHtmlMapper Assertions.assertEquals(2, metadata.getValues("concept_upper").length); Assertions.assertTrue(metadata.getFirstValue("fallback").contains("Main Title")); } + @Test + void testHtml() throws IOException { + checkHtml(new HashMap<>()); + } + + /** The IdentityHtmlMapper used by default must be applied in the forked JVM too. */ + @Test + @Timeout(60) + void testHtmlWithPipes() throws IOException { + Map conf = new HashMap<>(); + conf.put(ParserBolt.PARSE_TIMEOUT_PARAM, 30_000L); + conf.put(ParserBolt.PIPES_JVM_ARGS_PARAM, "-Xmx256m"); + checkHtml(conf); + } + @Test void testPdf() throws IOException { prepare("test.pdf.jsoupfilters.json", new HashMap<>()); From fc8cd7a473f11d9a5e9a99987848ffeb2817e26c Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Fri, 23 Oct 2026 09:01:57 +0100 Subject: [PATCH 3/4] JSoupBuilder: rename qName to satisfy checkstyle Co-Authored-By: Claude Opus 5.5 --- .../java/org/apache/stormcrawler/tika/JSoupBuilder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java index 11e90debd..550d701f3 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/JSoupBuilder.java @@ -55,9 +55,9 @@ public Document getDocument() { } @Override - public void startElement(String uri, String localName, String qName, Attributes atts) { + public void startElement(String uri, String localName, String qualifiedName, Attributes atts) { flushText(); - String name = StringUtils.isNotEmpty(localName) ? localName : qName; + String name = StringUtils.isNotEmpty(localName) ? localName : qualifiedName; Element element = new Element(Tag.valueOf(name, ParseSettings.htmlDefault), document.baseUri()); for (int i = 0; i < atts.getLength(); i++) { @@ -75,7 +75,7 @@ public void startElement(String uri, String localName, String qName, Attributes } @Override - public void endElement(String uri, String localName, String qName) { + public void endElement(String uri, String localName, String qualifiedName) { flushText(); if (current != document) { current = current.parent(); From f005931aeaa507ebf19a37e943d1031de33d687d Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Fri, 23 Oct 2026 13:31:04 +0100 Subject: [PATCH 4/4] Tika ParserBolt: fix prefixed attributes and unregistered HtmlMapper under Tika Pipes - Re-read the XML returned by the fork without namespace processing and restore the namespaces with NamespaceRestoringHandler: the fork writes names kept by the IdentityHtmlMapper, such as xlink:href or x-on:click, without declaring their prefix, which failed the whole document. - Only send the IdentityHtmlMapper to the fork when it is registered with Tika, otherwise log a warning and let the fork use its DefaultHtmlMapper instead of failing every parse. - configuration.adoc and README: describe which HtmlMapper the fork uses. Co-Authored-By: Claude Opus 5.5 --- docs/src/main/asciidoc/configuration.adoc | 2 +- external/tika/README.md | 2 +- .../tika/NamespaceRestoringHandler.java | 107 +++++++++++++++ .../apache/stormcrawler/tika/ParserBolt.java | 63 +++++++-- .../tika/NamespaceRestoringHandlerTest.java | 122 ++++++++++++++++++ .../tika/ParserBoltPipesHtmlMapperTest.java | 58 +++++++++ .../tika/ParserBoltPipesTimeoutTest.java | 32 +++++ 7 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 external/tika/src/main/java/org/apache/stormcrawler/tika/NamespaceRestoringHandler.java create mode 100644 external/tika/src/test/java/org/apache/stormcrawler/tika/NamespaceRestoringHandlerTest.java create mode 100644 external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesHtmlMapperTest.java diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index f628335b7..0889617c2 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -570,7 +570,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/tika[tika | parser.extract.embedded | false | Whether to parse embedded documents. Since Tika 4 embedded documents are no longer parsed unless this is set to `true`. | parser.tika.text.maxlength | -1 | Maximum number of characters of text extracted from a document, -1 (or any negative value) for no limit. When the limit is reached the parse stops, the text and outlinks extracted so far are kept and the metadata `parse.text.trimmed` is set to `true`. | parser.tika.pdf.maxpages | -1 | Maximum number of pages of a PDF document to parse, 0 or less to keep the `maxPages` of the Tika configuration (no limit by default). The pages after it are skipped, their text and outlinks are not extracted. Other settings of `pdf-parser` in the `parse-context` section of the Tika configuration are kept. -| parser.tika.timeout | -1 | Maximum time in milliseconds a document may take to parse, 0 or less for no limit. When set, the parse runs in a forked JVM (Tika Pipes), one per bolt instance; a parse that exceeds this is killed outright and reported to the status stream as an `ERROR` with the message `parse timeout`. A partial result returned between embedded documents is kept and marked `parse.text.trimmed`. When `parser.tika.text.maxlength` is negative (the default), the text is limited to 20,000,000 characters in this mode. Keep it well below `topology.message.timeout.secs`: the time a tuple waits behind a slow parse or a fork restart counts against that too. `parser.htmlmapper.classname` is ignored in this mode, see the README of the module. +| parser.tika.timeout | -1 | Maximum time in milliseconds a document may take to parse, 0 or less for no limit. When set, the parse runs in a forked JVM (Tika Pipes), one per bolt instance; a parse that exceeds this is killed outright and reported to the status stream as an `ERROR` with the message `parse timeout`. A partial result returned between embedded documents is kept and marked `parse.text.trimmed`. When `parser.tika.text.maxlength` is negative (the default), the text is limited to 20,000,000 characters in this mode. Keep it well below `topology.message.timeout.secs`: the time a tuple waits behind a slow parse or a fork restart counts against that too. The forked JVM uses the `IdentityHtmlMapper` (the default), any other value of `parser.htmlmapper.classname` is ignored in this mode and Tika's `DefaultHtmlMapper` is used instead, see the README of the module. | parser.tika.pipes.jvmargs | -Xmx512m | JVM arguments passed to each forked process under `parser.tika.timeout`. `-Xmx512m` is added when they set no maximum heap (`-Xmx`, `-XX:MaxHeapSize` or `-XX:MaxRAM*`). | parser.tika.pipes.maxfilesperprocess | - | Restart a forked process after this many documents under `parser.tika.timeout`, to bound slow leaks in parsing libraries. Unset uses Tika's default, 10000. | parser.tika.pipes.plugins.dir | - | Directory holding Tika Pipes plugin zips under `parser.tika.timeout`, not needed by default. Unset uses Tika's default plugin directory resolution. diff --git a/external/tika/README.md b/external/tika/README.md index 728c371b7..cbbad1400 100644 --- a/external/tika/README.md +++ b/external/tika/README.md @@ -49,6 +49,6 @@ Each `ParserBolt` instance runs one fork, so a host needs the worker heap plus o The JSoupFilters configured with `jsoup.filters.config.file` are applied to every document parsed by Tika, not only HTML, the same way as with the `JSoupParserBolt`; they run before the parse filters. The JSoup document is built from the XHTML Tika produces, not from the original content: a PDF becomes `

` and `

` elements, and the `` of an HTML page holds Tika's metadata (e.g. `` instead of ``), so expressions written for the original HTML may need adjusting. Element names are lowercased. The document is only built if at least one JSoupFilter is configured. -Under `parser.tika.timeout`, the default `IdentityHtmlMapper` is passed to the forked JVM, so the JSoup and parse filters get the same documents as with an in-process parse. Tika can only send the components registered in a `META-INF/tika/*.idx` file to the forked JVM: this module registers `IdentityHtmlMapper` in its `META-INF/tika/parse-context.idx`, which must be merged with the ones from Tika when building a shaded jar (see the `AppendingTransformer` entries in the archetype's `pom.xml`). Any other value of `parser.htmlmapper.classname` is ignored in this mode and the forked JVM uses Tika's `DefaultHtmlMapper`, which drops the elements it does not consider safe. +Under `parser.tika.timeout`, the default `IdentityHtmlMapper` is passed to the forked JVM, so the JSoup and parse filters get the same documents as with an in-process parse. Tika can only send the components registered in a `META-INF/tika/*.idx` file to the forked JVM: this module registers `IdentityHtmlMapper` in its `META-INF/tika/parse-context.idx`, which must be merged with the ones from Tika when building a shaded jar (see the `AppendingTransformer` entries in the archetype's `pom.xml`). If it is missing, the bolt logs a warning and the forked JVM uses Tika's `DefaultHtmlMapper`. Any other value of `parser.htmlmapper.classname` is ignored in this mode and the forked JVM uses Tika's `DefaultHtmlMapper`, which drops the elements it does not consider safe. Since Tika 4, Tika metadata keys use namespaced names, which surface as renamed `parse.*` keys, e.g. `parse.resourceName` is now `parse.tk:resource-name`. diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/NamespaceRestoringHandler.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/NamespaceRestoringHandler.java new file mode 100644 index 000000000..cf89f28e7 --- /dev/null +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/NamespaceRestoringHandler.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import java.util.Enumeration; +import org.apache.commons.lang3.StringUtils; +import org.apache.tika.sax.ContentHandlerDecorator; +import org.xml.sax.Attributes; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.AttributesImpl; +import org.xml.sax.helpers.NamespaceSupport; + +/** + * Resolves the namespaces of the events produced by a SAX parser which is not namespace aware, so + * that the XML returned by a Tika Pipes fork reaches the handlers as an in-process parse would. The + * fork writes the names kept by the IdentityHtmlMapper as they are, e.g. {@code xlink:href} or + * {@code x-on:click}, without declaring their prefix, which a namespace aware parser rejects. A + * name whose prefix is not declared is kept whole as the local name, an element without a declared + * prefix gets the default namespace in scope. + */ +class NamespaceRestoringHandler extends ContentHandlerDecorator { + + private static final String XMLNS = "xmlns"; + + private final NamespaceSupport namespaces = new NamespaceSupport(); + + private final String[] parts = new String[3]; + + NamespaceRestoringHandler(ContentHandler handler) { + super(handler); + } + + @Override + public void startElement(String uri, String localName, String qualifiedName, Attributes atts) + throws SAXException { + namespaces.pushContext(); + for (int i = 0; i < atts.getLength(); i++) { + String prefix = declaredPrefix(atts.getQName(i)); + if (prefix != null) { + namespaces.declarePrefix(prefix, atts.getValue(i)); + super.startPrefixMapping(prefix, atts.getValue(i)); + } + } + + AttributesImpl resolved = new AttributesImpl(); + for (int i = 0; i < atts.getLength(); i++) { + String name = atts.getQName(i); + if (declaredPrefix(name) != null) { + continue; + } + if (namespaces.processName(name, parts, true) != null) { + resolved.addAttribute(parts[0], parts[1], name, atts.getType(i), atts.getValue(i)); + } else { + resolved.addAttribute("", name, name, atts.getType(i), atts.getValue(i)); + } + } + + resolveElement(qualifiedName); + super.startElement(parts[0], parts[1], qualifiedName, resolved); + } + + @Override + public void endElement(String uri, String localName, String qualifiedName) throws SAXException { + resolveElement(qualifiedName); + super.endElement(parts[0], parts[1], qualifiedName); + Enumeration declared = namespaces.getDeclaredPrefixes(); + while (declared.hasMoreElements()) { + super.endPrefixMapping((String) declared.nextElement()); + } + namespaces.popContext(); + } + + /** Sets the namespace and local name of an element in {@link #parts}. */ + private void resolveElement(String qualifiedName) { + if (namespaces.processName(qualifiedName, parts, false) == null) { + parts[0] = StringUtils.defaultString(namespaces.getURI("")); + parts[1] = qualifiedName; + } + } + + /** Returns the prefix declared by an attribute, "" for the default namespace, or null. */ + private static String declaredPrefix(String attributeName) { + if (attributeName.equals(XMLNS)) { + return ""; + } + if (attributeName.startsWith(XMLNS + ":")) { + return attributeName.substring(XMLNS.length() + 1); + } + return null; + } +} diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 8bc3508cf..be57f7b36 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -74,6 +74,7 @@ import org.apache.tika.config.JsonConfig; import org.apache.tika.config.TimeoutLimits; import org.apache.tika.config.loader.TikaLoader; +import org.apache.tika.config.loader.TikaObjectMapperFactory; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.exception.TikaException; import org.apache.tika.exception.WriteLimitReachedException; @@ -97,6 +98,7 @@ import org.apache.tika.sax.LinkContentHandler; import org.apache.tika.sax.TeeContentHandler; import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.serialization.ComponentNameResolver; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.w3c.dom.DocumentFragment; @@ -170,7 +172,9 @@ public class ParserBolt extends BaseRichBolt { private static SAXParserFactory newHardenedSaxParserFactory() { SAXParserFactory factory = SAXParserFactory.newInstance(); - factory.setNamespaceAware(true); + // the fork writes prefixed names without declaring their namespace, see + // NamespaceRestoringHandler + factory.setNamespaceAware(false); try { factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); factory.setFeature("http://xml.org/sax/features/external-general-entities", false); @@ -644,11 +648,38 @@ private PipesForkParser buildPipesForkParser(Map conf) { } pipesConfig.setJvmArgs(forkedJvmArgs(conf)); - // Tika only sends the components registered in its index files to the forked JVM, - // which falls back to its default mapper otherwise - if (IdentityHtmlMapper.class.equals(htmlMapperClass)) { - pipesHtmlMapper = IdentityHtmlMapper.INSTANCE; - } else if (!DefaultHtmlMapper.class.equals(htmlMapperClass)) { + pipesHtmlMapper = pipesHtmlMapper(htmlMapperClass, isIdentityMapperRegistered()); + + try { + return new PipesForkParser(pipesConfig); + } catch (IOException | TikaConfigException e) { + throw new IllegalStateException( + "Failed to initialise the Tika Pipes fork parser for " + PARSE_TIMEOUT_PARAM, + e); + } + } + + /** Whether a META-INF/tika/parse-context.idx file registers the IdentityHtmlMapper. */ + static boolean isIdentityMapperRegistered() { + // Tika loads the registries when it creates its object mapper + TikaObjectMapperFactory.getMapper(); + return ComponentNameResolver.getFriendlyName(IdentityHtmlMapper.class) != null; + } + + /** + * Returns the HtmlMapper to send to the forked JVM, or null to leave it to Tika's {@link + * DefaultHtmlMapper}. Tika only sends the components registered in its index files, a mapper + * which is not registered would make every parse fail in the fork. + * + * @param identityMapperRegistered whether META-INF/tika/parse-context.idx registers the + * IdentityHtmlMapper + */ + static HtmlMapper pipesHtmlMapper( + Class htmlMapperClass, boolean identityMapperRegistered) { + if (DefaultHtmlMapper.class.equals(htmlMapperClass)) { + return null; + } + if (!IdentityHtmlMapper.class.equals(htmlMapperClass)) { LOG.warn( "parser.htmlmapper.classname ({}) is ignored under {}: the forked JVM maps" + " HTML with {}, which drops some elements from the documents given" @@ -656,15 +687,19 @@ private PipesForkParser buildPipesForkParser(Map conf) { htmlMapperClass.getName(), PARSE_TIMEOUT_PARAM, DefaultHtmlMapper.class.getSimpleName()); + return null; } - - try { - return new PipesForkParser(pipesConfig); - } catch (IOException | TikaConfigException e) { - throw new IllegalStateException( - "Failed to initialise the Tika Pipes fork parser for " + PARSE_TIMEOUT_PARAM, - e); + if (!identityMapperRegistered) { + LOG.warn( + "{} is not registered with Tika, the forked JVM used under {} maps HTML with {}" + + " instead. If the topology runs from a shaded jar, merge" + + " META-INF/tika/parse-context.idx with an AppendingTransformer.", + IdentityHtmlMapper.class.getSimpleName(), + PARSE_TIMEOUT_PARAM, + DefaultHtmlMapper.class.getSimpleName()); + return null; } + return IdentityHtmlMapper.INSTANCE; } /** @@ -851,7 +886,7 @@ private static void reparseIntoHandler(String xml, ContentHandler handler) throws SAXException, IOException { try { XMLReader reader = PIPES_CONTENT_PARSER_FACTORY.newSAXParser().getXMLReader(); - reader.setContentHandler(handler); + reader.setContentHandler(new NamespaceRestoringHandler(handler)); reader.parse(new InputSource(new StringReader(xml))); } catch (ParserConfigurationException e) { throw new IllegalStateException( diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/NamespaceRestoringHandlerTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/NamespaceRestoringHandlerTest.java new file mode 100644 index 000000000..56526ee0d --- /dev/null +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/NamespaceRestoringHandlerTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import javax.xml.parsers.SAXParserFactory; +import org.apache.tika.sax.XHTMLContentHandler; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; + +class NamespaceRestoringHandlerTest { + + /** Records the elements and attributes received, as uri|localName|qualifiedName. */ + private static final class Recorder extends DefaultHandler { + final List events = new ArrayList<>(); + + @Override + public void startPrefixMapping(String prefix, String uri) { + events.add("prefix " + prefix + "=" + uri); + } + + @Override + public void endPrefixMapping(String prefix) { + events.add("/prefix " + prefix); + } + + @Override + public void startElement( + String uri, String localName, String qualifiedName, Attributes atts) { + events.add(uri + "|" + localName + "|" + qualifiedName); + for (int i = 0; i < atts.getLength(); i++) { + events.add( + "@" + + atts.getURI(i) + + "|" + + atts.getLocalName(i) + + "|" + + atts.getQName(i) + + "=" + + atts.getValue(i)); + } + } + + @Override + public void endElement(String uri, String localName, String qualifiedName) { + events.add("/" + uri + "|" + localName + "|" + qualifiedName); + } + } + + private static List parse(String xml) throws Exception { + SAXParserFactory factory = SAXParserFactory.newInstance(); + factory.setNamespaceAware(false); + XMLReader reader = factory.newSAXParser().getXMLReader(); + Recorder recorder = new Recorder(); + reader.setContentHandler(new NamespaceRestoringHandler(recorder)); + reader.parse(new InputSource(new StringReader(xml))); + return recorder.events; + } + + @Test + void testUndeclaredPrefixes() throws Exception { + String xhtml = XHTMLContentHandler.XHTML; + List events = + parse( + "" + + ""); + Assertions.assertEquals( + List.of( + "prefix =" + xhtml, + xhtml + "|html|html", + xhtml + "|body|body", + xhtml + "|use|use", + "@|xlink:href|xlink:href=#i", + "@|x-on:click|x-on:click=go", + "/" + xhtml + "|use|use", + xhtml + "|o:p|o:p", + "@http://www.w3.org/XML/1998/namespace|lang|xml:lang=en", + "/" + xhtml + "|o:p|o:p", + "/" + xhtml + "|body|body", + "/" + xhtml + "|html|html", + "/prefix "), + events); + } + + @Test + void testDeclaredPrefix() throws Exception { + List events = parse(""); + Assertions.assertEquals( + List.of( + "prefix s=urn:s", + "|a|a", + "urn:s|b|s:b", + "@urn:s|c|s:c=1", + "/urn:s|b|s:b", + "/|a|a", + "/prefix s"), + events); + } +} diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesHtmlMapperTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesHtmlMapperTest.java new file mode 100644 index 000000000..1d260e6e3 --- /dev/null +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesHtmlMapperTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import org.apache.tika.parser.html.DefaultHtmlMapper; +import org.apache.tika.parser.html.HtmlMapper; +import org.apache.tika.parser.html.IdentityHtmlMapper; +import org.apache.tika.serialization.ComponentNameResolver; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Checks which HtmlMapper the ParserBolt sends to the Tika Pipes fork. */ +class ParserBoltPipesHtmlMapperTest { + + /** Registered by this module's META-INF/tika/parse-context.idx. */ + @Test + void testIdentityMapperIsRegistered() { + Assertions.assertTrue(ParserBolt.isIdentityMapperRegistered()); + Assertions.assertEquals( + "identity-html-mapper", + ComponentNameResolver.getFriendlyName(IdentityHtmlMapper.class)); + } + + @Test + void testIdentityMapper() { + Assertions.assertSame( + IdentityHtmlMapper.INSTANCE, + ParserBolt.pipesHtmlMapper(IdentityHtmlMapper.class, true)); + } + + /** Falls back to Tika's default, which the fork can always use, when not registered. */ + @Test + void testIdentityMapperNotRegistered() { + Assertions.assertNull(ParserBolt.pipesHtmlMapper(IdentityHtmlMapper.class, false)); + } + + @Test + void testOtherMappers() { + Assertions.assertNull(ParserBolt.pipesHtmlMapper(DefaultHtmlMapper.class, true)); + HtmlMapper custom = new DefaultHtmlMapper() {}; + Assertions.assertNull(ParserBolt.pipesHtmlMapper(custom.getClass(), true)); + } +} diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesTimeoutTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesTimeoutTest.java index 6ac37d91a..4b0c1e09d 100644 --- a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesTimeoutTest.java +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltPipesTimeoutTest.java @@ -131,6 +131,38 @@ void documentIsParsedUnderTimeout() throws IOException { Assertions.assertEquals("http://example.com/next", discovered.get(0).get(0)); } + /** + * The IdentityHtmlMapper keeps prefixed attributes such as {@code xlink:href} or {@code + * x-on:click}, which the fork writes back without declaring their namespace. + */ + @Test + @Timeout(30) + void prefixedAttributesUnderTimeout() throws IOException { + prepare(new HashMap<>()); + + String url = "https://example.org/svg.html"; + byte[] content = + ("" + + "" + + "next page" + + "") + .getBytes(StandardCharsets.UTF_8); + parse(url, content, new Metadata()); + + Assertions.assertTrue( + output.getEmitted(Constants.StatusStreamName).stream() + .noneMatch(t -> t.get(2) == Status.ERROR)); + List> emitted = output.getEmitted(); + Assertions.assertEquals(1, emitted.size()); + Assertions.assertTrue(emitted.get(0).get(3).toString().contains("next page")); + List> discovered = + output.getEmitted(Constants.StatusStreamName).stream() + .filter(t -> t.get(2) == Status.DISCOVERED) + .toList(); + Assertions.assertEquals(1, discovered.size()); + Assertions.assertEquals("http://example.com/next", discovered.get(0).get(0)); + } + /** * The fork stops at parser.tika.text.maxlength and returns XML cut off mid-document: the * document is still emitted, trimmed, instead of failing on the truncated XML.