diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index bd2b29380..0005f64a1 100644 --- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -28,7 +28,8 @@ config: fetcher.threads.number: 50 # override the JVM parameters for the workers - topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true" + # jdk.xml.* lift the JDK parser limits hit by large sitemaps and feeds; in local mode use STORM_JAR_JVM_OPTS + topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0" # mandatory when using Flux topology.kryo.register: diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index e5d888ccc..0d7628940 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -411,6 +411,8 @@ Configures parsing of fetched text and the handling of discovered URIs | urlfilters.config.file | urlfilters.json | JSON file defining URL filters. See link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json[default urlfilters.json]. |=== +The JDK XML parser fails on sitemaps, feeds and XML-based documents parsed by Tika with many entity references, such as escaped ampersands in URLs. Add `-Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0` to `topology.worker.childopts`, to `STORM_JAR_JVM_OPTS` in local mode, and to `parser.tika.pipes.jvmargs` when `parser.tika.timeout` is set. These properties set JVM-wide defaults for the JDK XML parsers; parser-specific settings take precedence. + ==== Metadata Options on how Storm Crawler should handle metadata tracking as well as diff --git a/docs/src/main/asciidoc/migrating.adoc b/docs/src/main/asciidoc/migrating.adoc index 7e07ec3b1..600fd5864 100644 --- a/docs/src/main/asciidoc/migrating.adoc +++ b/docs/src/main/asciidoc/migrating.adoc @@ -16,3 +16,9 @@ Core now depends only on HttpCore 5, not HttpClient. Applications using HttpClient classes through core's former transitive dependency must declare their own HttpClient dependency. The legacy OpenSearch module still uses HttpClient 4 through its REST client. + +Core no longer depends on Xerces, so sitemaps, feeds and the XML-based +documents parsed by Tika, such as OpenDocument files, use the JDK parser, +which fails on documents with many entity references. See +xref:configuration.adoc#_parsing[Parsing] for the JVM options that lift its +limits. Applications using Xerces classes must declare their own dependency. diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index 93315b477..70ad04c28 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -28,7 +28,8 @@ config: fetcher.threads.number: 50 # override the JVM parameters for the workers - topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true" + # jdk.xml.* lift the JDK parser limits hit by large sitemaps and feeds; in local mode use STORM_JAR_JVM_OPTS + topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0" # mandatory when using Flux topology.kryo.register: diff --git a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index 93315b477..70ad04c28 100644 --- a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -28,7 +28,8 @@ config: fetcher.threads.number: 50 # override the JVM parameters for the workers - topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true" + # jdk.xml.* lift the JDK parser limits hit by large sitemaps and feeds; in local mode use STORM_JAR_JVM_OPTS + topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0" # mandatory when using Flux topology.kryo.register: