#158 document the JDK XML limits for sitemaps and feeds

archetype/src/main/resources/archetype-resources/crawler-conf.yaml

@@ -28,7 +28,8 @@ config:
   fetcher.threads.number: 50
 
   # override the JVM parameters for the workers
-  topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true"
+  # jdk.xml.* lift the JDK parser limits hit by large sitemaps and feeds; in local mode use STORM_JAR_JVM_OPTS
+  topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0"
 
   # mandatory when using Flux
   topology.kryo.register:

docs/src/main/asciidoc/configuration.adoc

@@ -411,6 +411,8 @@ Configures parsing of fetched text and the handling of discovered URIs
 | urlfilters.config.file | urlfilters.json | JSON file defining URL filters. See link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json[default urlfilters.json].
 |===
 
+The JDK XML parser fails on sitemaps, feeds and XML-based documents parsed by Tika with many entity references, such as escaped ampersands in URLs. Add `-Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0` to `topology.worker.childopts`, to `STORM_JAR_JVM_OPTS` in local mode, and to `parser.tika.pipes.jvmargs` when `parser.tika.timeout` is set. These properties set JVM-wide defaults for the JDK XML parsers; parser-specific settings take precedence.
+
 ====  Metadata
 
 Options on how Storm Crawler should handle metadata tracking as well as

docs/src/main/asciidoc/migrating.adoc

@@ -16,3 +16,9 @@ Core now depends only on HttpCore 5, not HttpClient. Applications using
 HttpClient classes through core's former transitive dependency must declare
 their own HttpClient dependency. The legacy OpenSearch module still uses
 HttpClient 4 through its REST client.
+
+Core no longer depends on Xerces, so sitemaps, feeds and the XML-based
+documents parsed by Tika, such as OpenDocument files, use the JDK parser,
+which fails on documents with many entity references. See
+xref:configuration.adoc#_parsing[Parsing] for the JVM options that lift its
+limits. Applications using Xerces classes must declare their own dependency.

external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml

@@ -28,7 +28,8 @@ config:
   fetcher.threads.number: 50
 
   # override the JVM parameters for the workers
-  topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true"
+  # jdk.xml.* lift the JDK parser limits hit by large sitemaps and feeds; in local mode use STORM_JAR_JVM_OPTS
+  topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0"
 
   # mandatory when using Flux
   topology.kryo.register:

external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml

@@ -28,7 +28,8 @@ config:
   fetcher.threads.number: 50
 
   # override the JVM parameters for the workers
-  topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true"
+  # jdk.xml.* lift the JDK parser limits hit by large sitemaps and feeds; in local mode use STORM_JAR_JVM_OPTS
+  topology.worker.childopts: "-Xmx2g -Djava.net.preferIPv4Stack=true -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.totalEntitySizeLimit=0"
 
   # mandatory when using Flux
   topology.kryo.register: