diff --git a/core/pom.xml b/core/pom.xml index a44e157cf..92e35533b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -232,6 +232,12 @@ under the License. ${commons.lang.version} + + + org.apache.commons + commons-secure-xml + + com.squareup.okhttp3 diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java index f96a3f2df..2571e364b 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java @@ -34,8 +34,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.xml.secure.SecureDocumentBuilderFactory; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.filtering.URLFilter; import org.jetbrains.annotations.NotNull; @@ -164,7 +164,7 @@ private List readConfiguration(Reader reader) { // borrowed heavily from code in Configuration.java Document doc = - DocumentBuilderFactory.newInstance() + SecureDocumentBuilderFactory.newDefaultInstance() .newDocumentBuilder() .parse(new InputSource(reader)); Element root = doc.getDocumentElement(); diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java index 7dd5c417f..b21ee95a9 100644 --- a/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java @@ -29,7 +29,7 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; +import org.apache.commons.xml.secure.SecureXPathFactory; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.parse.JSoupFilter; import org.apache.stormcrawler.parse.ParseData; @@ -50,7 +50,7 @@ public class XPathFilter extends AbstractConfigurable implements JSoupFilter { protected final Map> expressions = new HashMap<>(); - private final XPath xpath = XPathFactory.newInstance().newXPath(); + private final XPath xpath = SecureXPathFactory.newDefaultInstance().newXPath(); /** * Supported extraction functions that can be appended to XPath expressions. These provide diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java index f1319baff..6d60395d3 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java @@ -33,6 +33,7 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.xml.secure.SecureXPathFactory; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.parse.ParseData; import org.apache.stormcrawler.parse.ParseFilter; @@ -72,7 +73,7 @@ public QName getReturnType() { private static final Logger LOG = LoggerFactory.getLogger(XPathFilter.class); - private XPathFactory factory = XPathFactory.newInstance(); + private XPathFactory factory = SecureXPathFactory.newDefaultInstance(); private XPath xpath = factory.newXPath(); protected final Map> expressions = new HashMap<>(); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/RegexURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/RegexURLNormalizerTest.java new file mode 100644 index 000000000..914edc4f7 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/filtering/RegexURLNormalizerTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.filtering; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.HashMap; +import org.apache.stormcrawler.filtering.regex.RegexURLNormalizer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class RegexURLNormalizerTest { + + private static final String URL = "https://example.com/page?sessionid=abc"; + + private static URLFilter createNormalizer(String rulesFile) { + ObjectNode params = new ObjectNode(JsonNodeFactory.instance); + params.put("regexNormalizerFile", rulesFile); + RegexURLNormalizer normalizer = new RegexURLNormalizer(); + normalizer.configure(new HashMap<>(), params); + return normalizer; + } + + @Test + void appliesRulesFromFile() { + URLFilter normalizer = createNormalizer("test-regex-normalizers-plain.xml"); + Assertions.assertEquals("https://example.com/page", normalizer.filter(null, null, URL)); + } + + /** + * The rules file declares an external DTD on a port nothing listens on. It must not be fetched: + * a parser that tried would fail and load no rules at all. + */ + @Test + void externalDtdIsNotFetched() { + URLFilter normalizer = createNormalizer("test-regex-normalizers-dtd.xml"); + Assertions.assertEquals("https://example.com/page", normalizer.filter(null, null, URL)); + } +} diff --git a/core/src/test/resources/test-regex-normalizers-dtd.xml b/core/src/test/resources/test-regex-normalizers-dtd.xml new file mode 100644 index 000000000..afce873fa --- /dev/null +++ b/core/src/test/resources/test-regex-normalizers-dtd.xml @@ -0,0 +1,24 @@ + + + + + + [?&]sessionid=[^&]* + + + diff --git a/core/src/test/resources/test-regex-normalizers-plain.xml b/core/src/test/resources/test-regex-normalizers-plain.xml new file mode 100644 index 000000000..6cf87b79f --- /dev/null +++ b/core/src/test/resources/test-regex-normalizers-plain.xml @@ -0,0 +1,23 @@ + + + + + [?&]sessionid=[^&]* + + + diff --git a/pom.xml b/pom.xml index 6fd674320..5c5ddb421 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ under the License. 2.22.0 1.28.0 1.23.0 + 1.1.0 33.7.2-jre 6.2 1.21.4 @@ -737,6 +738,12 @@ under the License. ${commons.codec.version} + + org.apache.commons + commons-secure-xml + ${commons.secure.xml.version} + +