From 32bab4a1e5aeca2294a8e0a749e24849ec8316ed Mon Sep 17 00:00:00 2001 From: Eunbin Son Date: Thu, 10 Sep 2026 09:49:49 +0900 Subject: [PATCH 1/3] Fix javadoc crawler reading a Maven Central index that stopped updating --- .../javadocs/JavaDocsCrawler.java | 57 ++++++++----------- .../javadocs/JavaDocsCrawlerTest.java | 52 +++++++++++------ 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java b/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java index c3f919474ec..40098321039 100644 --- a/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java +++ b/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java @@ -40,9 +40,11 @@ public final class JavaDocsCrawler { "io.opentelemetry.proto", "1.10.0"); private static final String MAVEN_CENTRAL_BASE_URL = - "https://search.maven.org/solrsearch/select?q=g:"; + "https://central.sonatype.com/solrsearch/select?q=g:"; private static final String JAVA_DOCS_BASE_URL = "https://javadoc.io/doc/"; - private static final int PAGE_SIZE = 20; + // Every group is fetched in a single request, so no paging parameter is sent. The largest group + // currently holds fewer than 200 artifacts and getArtifacts fails if the response is incomplete. + private static final int MAX_ROWS = 500; private static final int THROTTLE_MS = 500; // visible for testing @@ -81,30 +83,26 @@ public static void main(String[] args) throws Exception { static List getArtifacts(HttpClient client, String group) throws IOException, InterruptedException { - int start = 0; - Integer numFound; - List result = new ArrayList<>(); - - do { - if (start != 0) { - Thread.sleep(THROTTLE_MS); // try not to DDoS the site, it gets knocked over easily - } - - Map map = queryMavenCentral(client, group, start); - - numFound = - Optional.ofNullable(map) - .map(mavenResult -> (Map) mavenResult.get("response")) - .map(response -> (Integer) response.get("numFound")) - .orElse(null); - - List artifacts = convertToArtifacts(map); - result.addAll(artifacts); - - start += PAGE_SIZE; - } while (numFound != null && start < numFound); + Map map = queryMavenCentral(client, group, MAX_ROWS); + + Integer numFound = + Optional.ofNullable(map) + .map(mavenResult -> (Map) mavenResult.get("response")) + .map(response -> (Integer) response.get("numFound")) + .orElse(null); + + List artifacts = convertToArtifacts(map); + if (numFound != null && artifacts.size() < numFound) { + throw new IOException( + String.format( + Locale.ROOT, + "Received %d of %d artifacts for group %s, raise MAX_ROWS", + artifacts.size(), + numFound, + group)); + } - return result; + return artifacts; } private static List convertToArtifacts(Map map) { @@ -127,17 +125,12 @@ private static List convertToArtifacts(Map map) { .orElseGet(ArrayList::new); } - private static Map queryMavenCentral(HttpClient client, String group, int start) + private static Map queryMavenCentral(HttpClient client, String group, int rows) throws IOException, InterruptedException { URI uri = URI.create( String.format( - Locale.ROOT, - "%s%s&rows=%d&start=%d&wt=json", - MAVEN_CENTRAL_BASE_URL, - group, - PAGE_SIZE, - start)); + Locale.ROOT, "%s%s&rows=%d&wt=json", MAVEN_CENTRAL_BASE_URL, group, rows)); HttpRequest request = HttpRequest.newBuilder(uri).GET().build(); diff --git a/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java b/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java index 92150e0949f..230fbfed00f 100644 --- a/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java +++ b/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java @@ -7,6 +7,8 @@ import static io.opentelemetry.javadocs.JavaDocsCrawler.JAVA_DOC_DOWNLOADED_TEXT; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -32,12 +34,12 @@ class JavaDocsCrawlerTest { @Mock HttpResponse mockJavaDocResponse; @Test - void testGetArtifactsHandlesPagination() throws IOException, InterruptedException { - String page1Response = + void testGetArtifactsUsesASingleRequest() throws IOException, InterruptedException { + String response = """ { "response": { - "numFound": 40, + "numFound": 2, "docs": [ {"g": "group", "a": "artifact1", "latestVersion": "1.0"}, {"g": "group", "a": "artifact2", "latestVersion": "1.1"} @@ -45,32 +47,50 @@ void testGetArtifactsHandlesPagination() throws IOException, InterruptedExceptio } } """; - String page2Response = + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(HttpRequest.class); + + when(mockMavenCentralRequest1.body()).thenReturn(response); + when(mockMavenCentralRequest1.statusCode()).thenReturn(200); + + when(mockClient.send(any(), any())).thenReturn(mockMavenCentralRequest1); + + List artifacts = JavaDocsCrawler.getArtifacts(mockClient, "io.opentelemetry"); + + verify(mockClient, times(1)).send(requestCaptor.capture(), any()); + + String uri = requestCaptor.getValue().uri().toString(); + assertThat(uri) + .startsWith("https://central.sonatype.com/solrsearch/select?q=g:io.opentelemetry"); + assertThat(uri).contains("rows=500"); + assertThat(uri).doesNotContain("start="); + assertThat(artifacts) + .extracting(Artifact::getGroup, Artifact::getName, Artifact::getVersion) + .containsExactly(tuple("group", "artifact1", "1.0"), tuple("group", "artifact2", "1.1")); + } + + @Test + void testGetArtifactsFailsOnIncompleteResponse() throws IOException, InterruptedException { + String response = """ { "response": { "numFound": 40, "docs": [ - {"g": "group", "a": "artifact3", "latestVersion": "2.0"} + {"g": "group", "a": "artifact1", "latestVersion": "1.0"}, + {"g": "group", "a": "artifact2", "latestVersion": "1.1"} ] } } """; - when(mockMavenCentralRequest1.body()).thenReturn(page1Response); - when(mockMavenCentralRequest1.statusCode()).thenReturn(200); - when(mockMavenCentralRequest2.body()).thenReturn(page2Response); + when(mockMavenCentralRequest2.body()).thenReturn(response); when(mockMavenCentralRequest2.statusCode()).thenReturn(200); - when(mockClient.send(any(), any())) - .thenReturn(mockMavenCentralRequest1) - .thenReturn(mockMavenCentralRequest2); - - List artifacts = JavaDocsCrawler.getArtifacts(mockClient, "io.opentelemetry"); + when(mockClient.send(any(), any())).thenReturn(mockMavenCentralRequest2); - // 2 calls for the pagination - verify(mockClient, times(2)).send(any(), any()); - assertThat(artifacts.size()).isEqualTo(3); + assertThatThrownBy(() -> JavaDocsCrawler.getArtifacts(mockClient, "io.opentelemetry")) + .isInstanceOf(IOException.class) + .hasMessageContaining("io.opentelemetry"); } @Test From 8766e5d2154b8651aaeef167aa8dce12ad282c77 Mon Sep 17 00:00:00 2001 From: Eunbin Son <58901024+thswlsqls@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:15:04 +0900 Subject: [PATCH 2/3] Update javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java Co-authored-by: Jay DeLuca --- .../io/opentelemetry/javadocs/JavaDocsCrawler.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java b/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java index 40098321039..496680fc4db 100644 --- a/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java +++ b/javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java @@ -85,14 +85,21 @@ static List getArtifacts(HttpClient client, String group) throws IOException, InterruptedException { Map map = queryMavenCentral(client, group, MAX_ROWS); - Integer numFound = + // a 200 response without numFound means the endpoint changed shape; failing here keeps that + // from silently disabling the completeness check below + int numFound = Optional.ofNullable(map) .map(mavenResult -> (Map) mavenResult.get("response")) .map(response -> (Integer) response.get("numFound")) - .orElse(null); + .orElseThrow( + () -> + new IOException( + "Maven Central response for group " + + group + + " did not contain response.numFound")); List artifacts = convertToArtifacts(map); - if (numFound != null && artifacts.size() < numFound) { + if (artifacts.size() < numFound) { throw new IOException( String.format( Locale.ROOT, From e8cce372d4b8f089c9c59cea7522988d3c0a1db2 Mon Sep 17 00:00:00 2001 From: Eunbin Son Date: Sat, 12 Sep 2026 19:29:47 +0900 Subject: [PATCH 3/3] Add a test for a Maven Central response without numFound getArtifacts now fails when a 200 response has no response.numFound, so that a changed endpoint shape cannot silently skip the completeness check. Cover that path with a test that expects an IOException naming numFound and the group. The test fails against the previous code. Generated-by: Claude Code --- .../javadocs/JavaDocsCrawlerTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java b/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java index 230fbfed00f..5b12d3b1439 100644 --- a/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java +++ b/javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java @@ -93,6 +93,30 @@ void testGetArtifactsFailsOnIncompleteResponse() throws IOException, Interrupted .hasMessageContaining("io.opentelemetry"); } + @Test + void testGetArtifactsFailsWithoutNumFound() throws IOException, InterruptedException { + String response = + """ + { + "response": { + "docs": [ + {"g": "group", "a": "artifact1", "latestVersion": "1.0"} + ] + } + } + """; + + when(mockMavenCentralRequest1.body()).thenReturn(response); + when(mockMavenCentralRequest1.statusCode()).thenReturn(200); + + when(mockClient.send(any(), any())).thenReturn(mockMavenCentralRequest1); + + assertThatThrownBy(() -> JavaDocsCrawler.getArtifacts(mockClient, "io.opentelemetry")) + .isInstanceOf(IOException.class) + .hasMessageContaining("numFound") + .hasMessageContaining("io.opentelemetry"); + } + @Test void testCrawler() throws IOException, InterruptedException { Artifact artifact = new Artifact("io.opentelemetry", "opentelemetry-context", "1.49.0");