From d8f06aab9d45dba03b86788f7f9bd75d87eee717 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Fri, 31 Jul 2026 09:27:47 +0300 Subject: [PATCH 1/7] Adding upload statistics and alerts for both app binary uploads towards the CF API and OS uploads LMCROSSITXSADEPLOY-3054 --- .../src/main/java/module-info.java | 1 + .../controller/persistence/Messages.java | 3 + .../monitoring/UploadDurationTracker.java | 99 +++++++++++++++++ .../monitoring/UploadTimeoutMatcher.java | 31 ++++++ .../services/AzureObjectStoreFileStorage.java | 19 +++- .../services/GcpObjectStoreFileStorage.java | 18 ++- .../JCloudsObjectStoreFileStorage.java | 21 +++- .../AzureObjectStoreFileStorageTest.java | 6 +- .../GcpObjectStoreFileStorageTest.java | 6 +- .../JCloudsObjectStoreFileStorageTest.java | 18 ++- .../steps/UploadAppAsyncExecution.java | 22 +++- .../process/steps/UploadAppStep.java | 5 +- .../steps/UploadAppAsyncExecutionTest.java | 24 ++-- .../FileStorageConfiguration.java | 8 +- .../web/configuration/JmxConfiguration.java | 7 +- .../factory/ObjectStoreSelectorFactory.java | 15 ++- .../AppUploaderThreadPoolInformation.java | 28 +++++ .../web/monitoring/UploadDurationMetrics.java | 103 ++++++++++++++++++ .../web/monitoring/UploadMetricsMBean.java | 33 ++++++ .../ObjectStoreSelectorFactoryTest.java | 15 ++- 20 files changed, 438 insertions(+), 44 deletions(-) create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java create mode 100644 multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java create mode 100644 multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java create mode 100644 multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index b5078c3fc3..5924369930 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -9,6 +9,7 @@ exports org.cloudfoundry.multiapps.controller.persistence.model; exports org.cloudfoundry.multiapps.controller.persistence.model.adapter; exports org.cloudfoundry.multiapps.controller.persistence.model.filters; + exports org.cloudfoundry.multiapps.controller.persistence.monitoring; exports org.cloudfoundry.multiapps.controller.persistence.query; exports org.cloudfoundry.multiapps.controller.persistence.query.criteria; exports org.cloudfoundry.multiapps.controller.persistence.query.impl; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 3e44634d02..742d0cc43b 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -74,6 +74,9 @@ public final class Messages { // INFO log messages: public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}"; + public static final String TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for JClouds object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for GCP object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for Azure object store upload: {0} in millis"; // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java new file mode 100644 index 0000000000..d6ac887b8b --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java @@ -0,0 +1,99 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import jakarta.inject.Named; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +@Named +public class UploadDurationTracker { + + private final UploadPathStatistics appBinaryStatistics = new UploadPathStatistics(); + + private final UploadPathStatistics objectStoreStatistics = new UploadPathStatistics(); + + public void recordAppBinaryUpload(long durationMillis, boolean timedOut) { + appBinaryStatistics.record(durationMillis, timedOut); + } + + public void recordObjectStoreUpload(long durationMillis, boolean timedOut) { + objectStoreStatistics.record(durationMillis, timedOut); + } + + public void recordAppBinaryUploadRejection() { + appBinaryStatistics.recordRejection(); + } + + public UploadPathStatistics getAppBinaryStatistics() { + return this.appBinaryStatistics; + } + + public UploadPathStatistics getObjectStoreStatistics() { + return this.objectStoreStatistics; + } + + public static final class UploadPathStatistics { + + private final LongAdder total = new LongAdder(); + + private final LongAdder timeouts = new LongAdder(); + + private final LongAdder sumDuration = new LongAdder(); + + private final LongAdder rejections = new LongAdder(); + + private final AtomicLong rejectionsInWindow = new AtomicLong(0); + + private final AtomicLong maxDuration = new AtomicLong(0); + + private final AtomicLong timeoutsInWindow = new AtomicLong(0); + + public void record(long durationMillis, boolean timedOut) { + long duration = Math.max(0, durationMillis); + total.increment(); + + if (timedOut) { + timeouts.increment(); + timeoutsInWindow.incrementAndGet(); + } + + sumDuration.add(duration); + maxDuration.accumulateAndGet(duration, Math::max); + } + + public void recordRejection() { + rejections.increment(); + rejectionsInWindow.incrementAndGet(); + } + + public long totalCount() { + return total.sum(); + } + + public long timeoutCount() { + return timeouts.sum(); + } + + public long maxDurationMs() { + return maxDuration.getAndSet(0); + } + + public long sumDurationMs() { + return sumDuration.sum(); + } + + public long timeoutsInWindow() { + return timeoutsInWindow.getAndSet(0); + } + + public long rejectionCount() { + return rejections.sum(); + } + + public long rejectionsInWindow() { + return rejectionsInWindow.getAndSet(0); + } + + } + +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java new file mode 100644 index 0000000000..f229a53aee --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java @@ -0,0 +1,31 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import com.google.cloud.storage.StorageException; +import io.netty.handler.timeout.ReadTimeoutException; + +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; + +public class UploadTimeoutMatcher { + + private UploadTimeoutMatcher() { + + } + + public static boolean isUploadTimeoutException(Throwable throwable) { + if (throwable == null) { + return false; + } + + Throwable cause = throwable.getCause(); + while (cause != null) { + if (cause instanceof SocketTimeoutException || cause instanceof TimeoutException || cause instanceof ReadTimeoutException || ( + cause instanceof StorageException + && ((StorageException) cause).getCode() == 504)) { + return true; + } + cause = cause.getCause(); + } + return false; + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index 721e7234b8..6bc2799a2d 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -5,6 +5,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -29,6 +30,8 @@ import com.azure.storage.blob.options.BlobParallelUploadOptions; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -46,15 +49,18 @@ public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { private static final int MAX_CONCURRENCY = 5; private final HttpClient httpClient; private final BlobContainerClient containerClient; + private final UploadDurationTracker uploadDurationTracker; - public AzureObjectStoreFileStorage(Map credentials) { + public AzureObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.httpClient = new JdkHttpClientBuilder().build(); this.containerClient = createContainerClient(credentials); + this.uploadDurationTracker = uploadDurationTracker; } @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + LocalDateTime startTime = LocalDateTime.now(); try { ParallelTransferOptions pto = new ParallelTransferOptions().setMaxSingleUploadSizeLong(MAX_SINGLE_UPLOAD_SIZE) .setMaxConcurrency(MAX_CONCURRENCY) @@ -67,10 +73,14 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage blobClient.uploadWithResponse(blobParallelUploadOptions, ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, null); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileEntry.getSize() - .longValue())); + .longValue())); } catch (BlobStorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -218,6 +228,11 @@ private int removeBlobsByFilter(Predicate filter) { return deletedBlobsResult; } + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + protected Set getEntryNames(Predicate filter) { BlobListDetails blobListDetails = new BlobListDetails().setRetrieveMetadata(true); ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(blobListDetails); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 2a0757eb5f..c4d73c8711 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -13,6 +13,8 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -25,6 +27,7 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -41,10 +44,12 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; + private final UploadDurationTracker uploadDurationTracker; - public GcpObjectStoreFileStorage(Map credentials) { + public GcpObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.storage = createObjectStoreStorage(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected Storage createObjectStoreStorage(Map credentials) { @@ -92,11 +97,16 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { + LocalDateTime startTime = LocalDateTime.now(); try { storage.createFrom(blobInfo, content); } catch (IOException | StorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -283,4 +293,10 @@ private static final class CredentialKeys { static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; static final String BUCKET = "bucket"; } + + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 287aa5a344..e7bea41567 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.HashSet; import java.util.List; @@ -10,9 +11,12 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; + import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.jclouds.blobstore.BlobStore; @@ -37,10 +41,12 @@ public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage { private final BlobStore blobStore; private final String container; + private final UploadDurationTracker uploadDurationTracker; - public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { + public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container, UploadDurationTracker uploadDurationTracker) { this.blobStore = blobStore; this.container = container; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -55,13 +61,18 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentLength(fileSize) .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); + LocalDateTime startTime = LocalDateTime.now(); try { putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), fileEntry.getNamespace())); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -95,7 +106,7 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { int deletedFiles = removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, - namespace)); + namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @@ -260,4 +271,10 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } + + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java index 437981fa8a..adb3361faa 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -20,6 +20,7 @@ import com.azure.storage.blob.specialized.BlobInputStream; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -51,6 +52,9 @@ class AzureObjectStoreFileStorageTest { @Mock private BlobInputStream blobInputStream; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AzureObjectStoreFileStorage fileStorage; private InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private final String TEST_SPACE_ID = UUID.randomUUID() @@ -70,7 +74,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AzureObjectStoreFileStorage(Map.of()) { + fileStorage = new AzureObjectStoreFileStorage(Map.of(), uploadDurationTracker) { @Override protected BlobContainerClient createContainerClient(Map credentials) { diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 14264ffa72..6fb0f3cf44 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -8,6 +8,7 @@ import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,13 +37,14 @@ class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; private Storage mockedStorage; private GcpObjectStoreFileStorage mockedGcpFileStorage; + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @Override @BeforeEach public void setUp() { storage = LocalStorageHelper.getOptions() .getService(); - fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { @@ -54,7 +56,7 @@ protected Storage createObjectStoreStorage(Map credentials) { namespace = UUID.randomUUID() .toString(); mockedStorage = mock(Storage.class); - mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { return mockedStorage; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 896cf90ee1..77de41d591 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -16,10 +16,12 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; + import jakarta.xml.bind.DatatypeConverter; import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; @@ -28,6 +30,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; + import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -57,10 +60,12 @@ class JCloudsObjectStoreFileStorageTest { private BlobStoreContext blobStoreContext; + private UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); + @BeforeEach protected void setUp() { createBlobStoreContext(); - fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER, uploadDurationTracker) { @Override protected long getRetryWaitTime() { return 1; @@ -233,7 +238,7 @@ void testConnection() { @Test void testTestConnectionWhenContainerExists() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(true); @@ -244,7 +249,7 @@ void testTestConnectionWhenContainerExists() { @Test void testTestConnectionWhenContainerDoesNotExist() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(false); @@ -404,7 +409,7 @@ protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(true); @@ -418,7 +423,7 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { void existsInObjectStoreWithMockedBlobStoreFileNotFound() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(false); @@ -433,7 +438,8 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesCorrectContainer() { BlobStore mockBlobStore = mock(BlobStore.class); String testContainer = "test-container"; FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer, + uploadDurationTracker); when(mockBlobStore.blobExists(testContainer, fileEntry.getId())).thenReturn(true); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java index 7f83745815..14f13392c0 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java @@ -20,12 +20,14 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; import org.cloudfoundry.multiapps.controller.client.lib.domain.UploadStatusCallbackExtended; import org.cloudfoundry.multiapps.controller.core.Constants; +import org.cloudfoundry.multiapps.controller.core.cloudlogging.OperationLogsExporter; import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationEnvironmentUpdater; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration; -import org.cloudfoundry.multiapps.controller.core.cloudlogging.OperationLogsExporter; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.services.ProcessLoggerPersister; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.context.ApplicationToUploadContext; @@ -48,15 +50,17 @@ public class UploadAppAsyncExecution implements AsyncExecution { private final ApplicationConfiguration applicationConfiguration; private final ExecutorService appUploaderThreadPool; private final OperationLogsExporter operationLogsExporter; + private final UploadDurationTracker uploadDurationTracker; public UploadAppAsyncExecution(ApplicationZipBuilder applicationZipBuilder, ProcessLoggerPersister processLoggerPersister, ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool, - OperationLogsExporter operationLogsExporter) { + OperationLogsExporter operationLogsExporter, UploadDurationTracker uploadDurationTracker) { this.applicationZipBuilder = applicationZipBuilder; this.processLoggerPersister = processLoggerPersister; this.applicationConfiguration = applicationConfiguration; this.appUploaderThreadPool = appUploaderThreadPool; this.operationLogsExporter = operationLogsExporter; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -76,6 +80,7 @@ public AsyncExecutionState execute(ProcessContext context) { try { runningUpload = appUploaderThreadPool.submit(() -> doUpload(context, applicationToProcess, applicationToUploadContext)); } catch (RejectedExecutionException rejectedExecutionException) { + uploadDurationTracker.recordAppBinaryUploadRejection(); LOGGER.warn(rejectedExecutionException.getMessage(), rejectedExecutionException); context.getStepLogger() .warn(Messages.UPLOAD_OF_APPLICATION_0_WAS_NOT_ACCEPTED_BY_INSTANCE_1, applicationToProcess.getName(), @@ -86,13 +91,12 @@ public AsyncExecutionState execute(ProcessContext context) { try { cloudPackage = runningUpload.get(); } catch (InterruptedException | ExecutionException e) { + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new SLException(e, e.getMessage()); } - LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); - long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) - .toMillis(); + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), false); context.getStepLogger() - .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, timeElapsedForUpload); + .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(context)); return processCloudPackage(context, client, cloudPackage); } @@ -210,6 +214,12 @@ private void attemptToUpdateApplicationDigest(CloudControllerClient client, Clou newApplicationDigest); } + private long getElapsedTimeInMillis(ProcessContext context) { + LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + @Override public String getPollingErrorMessage(ProcessContext context) { CloudApplicationExtended appToProcess = context.getVariable(Variables.APP_TO_PROCESS); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java index 3b0cbaa25a..3874382dbb 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java @@ -21,6 +21,7 @@ import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationFileDigestDetector; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.security.serialization.DynamicSecureSerialization; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.security.util.SecureLoggingUtil; @@ -53,6 +54,8 @@ public class UploadAppStep extends TimeoutAsyncFlowableStep { protected CloudPackagesGetter cloudPackagesGetter; @Inject private ExecutorService appUploaderThreadPool; + @Inject + private UploadDurationTracker uploadDurationTracker; @Override public StepPhase executeAsyncStep(ProcessContext context) throws FileStorageException { @@ -186,7 +189,7 @@ private void removeApplicationDigestIfSet(ProcessContext context, Map getAsyncStepExecutions(ProcessContext context) { return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool, - operationLogsExporter), + operationLogsExporter, uploadDurationTracker), new PollUploadAppStatusExecution()); } diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java index 4e832ac010..d831d3b2f0 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java @@ -29,6 +29,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableLoggingConfiguration; import org.cloudfoundry.multiapps.controller.persistence.model.LogLevel; import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveIterator; @@ -92,6 +93,7 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest .build(); private final MtaArchiveElements mtaArchiveElements = new MtaArchiveElements(); private final ExecutorService appUploaderThreadPool = mock(ExecutorService.class); + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @TempDir Path tempDir; @@ -212,8 +214,9 @@ void testSendApplicationLogsToCloudLoggingService_sendsLogsForEachProcessLog() { context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, loggingConfiguration); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(List.of("log-1", "log-2")); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(List.of("log-1", "log-2")); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -230,8 +233,9 @@ void testSendApplicationLogsToCloudLoggingService_withNullLoggingConfiguration_s context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, null); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(List.of("log-1")); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(List.of("log-1")); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -248,8 +252,9 @@ void testSendApplicationLogsToCloudLoggingService_withNoLogs_doesNotSendProcessL context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, loggingConfiguration); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(Collections.emptyList()); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(Collections.emptyList()); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -266,8 +271,9 @@ void testSendApplicationLogsToCloudLoggingService_notCalledWhenStatusIsNotReady( context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, loggingConfiguration); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(List.of("process-log-content")); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(List.of("process-log-content")); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -314,7 +320,7 @@ protected List getAsyncStepExecutions(ProcessContext context) { getProcessLogsPersister(), configuration, appUploaderThreadPool, - operationLogsExporter) { + operationLogsExporter, uploadDurationTracker) { }); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java index c892373baa..85cdcb3c31 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java @@ -1,9 +1,11 @@ package org.cloudfoundry.multiapps.controller.web.configuration; +import jakarta.inject.Inject; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.NoRetryErrorClassifier; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.RetryableErrorClassifier; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.configuration.factory.ObjectStoreSelectorFactory; import org.springframework.context.annotation.Bean; @@ -16,10 +18,14 @@ public class FileStorageConfiguration { private static final String OBJECT_STORE_SERVICE_NAME = "deploy-service-os"; + @Inject + private UploadDurationTracker uploadDurationTracker; + @Bean public ObjectStoreSelectorFactory objectStoreSelector(EnvironmentServicesFinder vcapServiceFinder, ApplicationConfiguration applicationConfiguration) { - return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration); + return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration, + uploadDurationTracker); } @Bean diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java index cb52bb629a..a8aca8efc2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java @@ -3,6 +3,7 @@ import java.util.Map; import org.cloudfoundry.multiapps.controller.web.monitoring.Metrics; +import org.cloudfoundry.multiapps.controller.web.monitoring.UploadDurationMetrics; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jmx.export.MBeanExporter; @@ -12,10 +13,12 @@ public class JmxConfiguration { private static final String METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=MetricsMBean"; + private static final String UPLOAD_METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=UploadMetricsMBean"; + @Bean - public MBeanExporter jmxExporter(Metrics metrics) { + public MBeanExporter jmxExporter(Metrics metrics, UploadDurationMetrics uploadDurationMetrics) { MBeanExporter mBeanExporter = new MBeanExporter(); - mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics)); + mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics, UPLOAD_METRICS_BEAN, uploadDurationMetrics)); return mBeanExporter; } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java index 117ed389ad..d4a4d27284 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java @@ -7,11 +7,13 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; + import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; @@ -39,13 +41,16 @@ public class ObjectStoreSelectorFactory { private final EnvironmentServicesFinder environmentServicesFinder; private final ApplicationConfiguration applicationConfiguration; private final SelectedObjectStore selectedObjectStore; + private final UploadDurationTracker uploadDurationTracker; public ObjectStoreSelectorFactory(String serviceName, EnvironmentServicesFinder environmentServicesFinder, - ApplicationConfiguration applicationConfiguration) { + ApplicationConfiguration applicationConfiguration, + UploadDurationTracker uploadDurationTracker) { this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; this.applicationConfiguration = applicationConfiguration; this.selectedObjectStore = doSelect(); + this.uploadDurationTracker = uploadDurationTracker; } public FileStorage fileStorage() { @@ -155,15 +160,15 @@ private SelectedObjectStore selectFor(ObjectStoreServiceInfo objectStoreServiceI } protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -203,7 +208,7 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { return new JCloudsObjectStoreFileStorage(context.getBlobStore(), (String) objectStoreServiceInfo.getCredentials() - .get(Constants.BUCKET)); + .get(Constants.BUCKET), uploadDurationTracker); } private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java new file mode 100644 index 0000000000..3207495a23 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java @@ -0,0 +1,28 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +@Named +public class AppUploaderThreadPoolInformation { + + private final ThreadPoolExecutor appUploaderPool; + + @Inject + public AppUploaderThreadPoolInformation( + @Named("appUploaderThreadPool") ExecutorService appUploaderThreadPool) { + this.appUploaderPool = (ThreadPoolExecutor) appUploaderThreadPool; + } + + public int getActiveThreads() { + return appUploaderPool.getActiveCount(); + } + + public int getMaxThreads() { + return appUploaderPool.getMaximumPoolSize(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java new file mode 100644 index 0000000000..36e111505b --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java @@ -0,0 +1,103 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; + +@Named +public class UploadDurationMetrics implements UploadMetricsMBean { + + private final UploadDurationTracker tracker; + + private final AppUploaderThreadPoolInformation appUploaderInfo; + + @Inject + public UploadDurationMetrics(UploadDurationTracker tracker, + AppUploaderThreadPoolInformation appUploaderInfo) { + this.tracker = tracker; + this.appUploaderInfo = appUploaderInfo; + } + + @Override + public long getAppBinaryUploadTotalCount() { + return tracker.getAppBinaryStatistics() + .totalCount(); + } + + @Override + public long getAppBinaryUploadTimeoutCount() { + return tracker.getAppBinaryStatistics() + .timeoutCount(); + } + + @Override + public long getAppBinaryUploadMaxDurationMs() { + return tracker.getAppBinaryStatistics() + .maxDurationMs(); + } + + @Override + public long getAppBinaryUploadSumDurationMs() { + return tracker.getAppBinaryStatistics() + .sumDurationMs(); + } + + @Override + public int getAppUploaderActiveThreads() { + return appUploaderInfo.getActiveThreads(); + } + + @Override + public int getAppUploaderMaxThreads() { + return appUploaderInfo.getMaxThreads(); + } + + @Override + public long getAppUploaderRejectionCount() { + return tracker.getAppBinaryStatistics() + .rejectionCount(); + } + + @Override + public long getAppUploaderRejectionCountInWindow() { + return tracker.getAppBinaryStatistics() + .rejectionsInWindow(); + } + + @Override + public long getObjectStoreUploadTotalCount() { + return tracker.getObjectStoreStatistics() + .totalCount(); + } + + @Override + public long getObjectStoreUploadTimeoutCount() { + return tracker.getObjectStoreStatistics() + .timeoutCount(); + } + + @Override + public long getObjectStoreUploadMaxDurationMs() { + return tracker.getObjectStoreStatistics() + .maxDurationMs(); + } + + @Override + public long getObjectStoreUploadSumDurationMs() { + return tracker.getObjectStoreStatistics() + .sumDurationMs(); + } + + @Override + public long getAppBinaryUploadTimeoutsInWindow() { + return tracker.getAppBinaryStatistics() + .timeoutsInWindow(); + } + + @Override + public long getObjectStoreUploadTimeoutsInWindow() { + return tracker.getObjectStoreStatistics() + .timeoutsInWindow(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java new file mode 100644 index 0000000000..396c352db1 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java @@ -0,0 +1,33 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +public interface UploadMetricsMBean { + + long getAppBinaryUploadTotalCount(); + + long getAppBinaryUploadTimeoutCount(); + + long getAppBinaryUploadMaxDurationMs(); + + long getAppBinaryUploadSumDurationMs(); + + int getAppUploaderActiveThreads(); + + int getAppUploaderMaxThreads(); + + long getAppUploaderRejectionCount(); + + long getAppUploaderRejectionCountInWindow(); + + long getObjectStoreUploadTotalCount(); + + long getObjectStoreUploadTimeoutCount(); + + long getObjectStoreUploadMaxDurationMs(); + + long getObjectStoreUploadSumDurationMs(); + + long getAppBinaryUploadTimeoutsInWindow(); + + long getObjectStoreUploadTimeoutsInWindow(); + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java index b28570c678..911f956f89 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java @@ -4,6 +4,7 @@ import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -64,6 +65,8 @@ class ObjectStoreSelectorFactoryTest { private AwsS3ObjectStoreFileStorage awsS3ObjectStoreFileStorage; @Mock private AzureObjectStoreFileStorage azureObjectStoreFileStorage; + @Mock + private UploadDurationTracker uploadDurationTracker; private ObjectStoreServiceInfo capturedGcpServiceInfo; private ObjectStoreServiceInfo capturedAzureServiceInfo; @@ -103,7 +106,7 @@ void testObjectStoreCreationWhenEnvIsValidForAws() { assertInstanceOf(AwsS3ObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AwsTransientErrorClassifier.class, selector.classifier()); verify(awsS3ObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -116,7 +119,7 @@ void testObjectStoreCreationWhenEnvIsValidForAzure() { assertInstanceOf(AzureObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AzureTransientErrorClassifier.class, selector.classifier()); verify(azureObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -129,12 +132,12 @@ void testObjectStoreCreationWhenEnvIsValidForGcp() { assertInstanceOf(GcpObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(GcpTransientErrorClassifier.class, selector.classifier()); verify(gcpObjectStoreFileStorage) - .testConnection(); + .testConnection(); } static Stream testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid() { return Stream.of( - // @formatter:off + // @formatter:off // (0) Unknown provider name: Arguments.of("WRONG_PROVIDER"), // (1) Null env value: @@ -169,7 +172,7 @@ void testObjectStoreCreationFallsBackToNextProviderWhenFirstFails() { assertInstanceOf(JCloudsObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(JCloudsTransientErrorClassifier.class, selector.classifier()); verify(jCloudsObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -289,7 +292,7 @@ private class ObjectStoreSelectorFactoryMock extends ObjectStoreSelectorFactory ObjectStoreSelectorFactoryMock(String serviceName, EnvironmentServicesFinder environmentServicesFinder, ApplicationConfiguration applicationConfiguration) { - super(serviceName, environmentServicesFinder, applicationConfiguration); + super(serviceName, environmentServicesFinder, applicationConfiguration, uploadDurationTracker); } @Override From db7e61f3eda5d356ada0bfb6097bc164ceb207db Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Fri, 31 Jul 2026 12:23:09 +0300 Subject: [PATCH 2/7] Fix merge conflicts and add AWS OS upload tracking LMCROSSITXSADEPLOY-3054 --- .../src/main/java/module-info.java | 1 + .../controller/persistence/Messages.java | 1 + .../services/AwsS3ObjectStoreFileStorage.java | 24 ++++++++++++++-- .../AwsS3ObjectStoreFileStorageTest.java | 28 +++++++++++-------- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 5924369930..c3001f6a66 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -81,4 +81,5 @@ requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; + requires io.netty.handler; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 742d0cc43b..459531195f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -77,6 +77,7 @@ public final class Messages { public static final String TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for JClouds object store upload: {0} in millis"; public static final String TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for GCP object store upload: {0} in millis"; public static final String TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for Azure object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for AWS object store upload: {0} in millis"; // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index a075038c51..0647856341 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.net.URI; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -16,13 +17,17 @@ import java.util.concurrent.Executors; import java.util.function.BiPredicate; import java.util.function.Consumer; + import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cglib.core.Local; import org.springframework.http.MediaType; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -57,10 +62,12 @@ public class AwsS3ObjectStoreFileStorage extends ObjectStoreFileStorage { private final S3Client s3Client; private final String bucketName; + private final UploadDurationTracker uploadDurationTracker; - public AwsS3ObjectStoreFileStorage(Map credentials) { + public AwsS3ObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.s3Client = createS3Client(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected S3Client createS3Client(Map credentials) { @@ -103,6 +110,7 @@ protected ClientOverrideConfiguration buildClientOverrideConfig() { @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + LocalDateTime startTime = LocalDateTime.now(); long fileSize = fileEntry.getSize() .longValue(); PutObjectRequest request = PutObjectRequest.builder() @@ -115,8 +123,12 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage try { s3Client.putObject(request, RequestBody.fromInputStream(new BufferedInputStream(content), fileSize)); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } catch (Exception e) { LOGGER.error(MessageFormat.format(Messages.S3_UPLOAD_FAILED_FILE_0_SIZE_1, fileEntry.getName(), fileSize, e)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED, fileEntry.getName(), fileEntry.getNamespace()), e); } @@ -183,14 +195,15 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); + int deletedFiles = deleteByFilterAndCount( + (key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterByModificationTime(metadata, key, - modificationTime)); + modificationTime)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_MODIFIED_BEFORE_1, deletedFiles, modificationTime)); return deletedFiles; } @@ -319,6 +332,11 @@ public void destroy() { s3Client.close(); } + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + private static final class CredentialKeys { static final String ACCESS_KEY_ID = "access_key_id"; static final String SECRET_ACCESS_KEY = "secret_access_key"; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java index b5b5a08011..e41c95d561 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java @@ -2,6 +2,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -62,16 +63,19 @@ class AwsS3ObjectStoreFileStorageTest { @Mock private FileContentProcessor fileContentProcessor; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AwsS3ObjectStoreFileStorage fileStorage; private final InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private static final String TEST_SPACE_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_SPACE_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String NAMESPACE = "namespace"; private static final String NAMESPACE_2 = "namespace_2"; @@ -80,7 +84,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME)) { + fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME), uploadDurationTracker) { @Override protected S3Client createS3Client(Map credentials) { @@ -136,7 +140,7 @@ void testGetFileEntriesWithoutContentWithoutMatches() { } @Test - void testExistsInObjectStoreWhenFileExists() { + void testExistsInObjectStoreWhenFileExists() { when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() .build()); FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); @@ -232,7 +236,7 @@ void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { @Test void testDeleteFilesModifiedBefore() { LocalDateTime oldModified = FILE_TIMESTAMP - .minusMinutes(15); + .minusMinutes(15); setupListObjectsWithKeys(TEST_ID, TEST_ID_2); setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, oldModified); @@ -250,7 +254,7 @@ void testDeleteFilesModifiedBeforeWithNoOldFiles() { setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, FILE_TIMESTAMP); LocalDateTime cutoff = FILE_TIMESTAMP - .minusDays(1); + .minusDays(1); int deletedCount = fileStorage.deleteFilesModifiedBefore(cutoff); assertEquals(0, deletedCount); @@ -532,7 +536,7 @@ void testBuildClientOverrideConfigSetsApiCallAttemptTimeout() { assertTrue(config.apiCallAttemptTimeout() .isPresent()); assertEquals(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallAttemptTimeout() - .get()); + .get()); } @Test @@ -544,8 +548,8 @@ void testBuildClientOverrideConfigSetsRetryStrategy() { assertInstanceOf(StandardRetryStrategy.class, config.retryStrategy() .get()); assertEquals(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG, config.retryStrategy() - .get() - .maxAttempts()); + .get() + .maxAttempts()); } @Test @@ -555,7 +559,7 @@ void testCreateS3ClientWithValidCredentials() { "bucket", "test-bucket", "host", "s3.amazonaws.com", "region", "eu-central-1"); - AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials); + AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials, uploadDurationTracker); assertNotNull(storage); storage.destroy(); From d646bae1c5f6d701590d5a2d3e4ade7b13f8f910 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Sun, 2 Aug 2026 22:49:49 +0300 Subject: [PATCH 3/7] Fixing spring bean problems and imports LMCROSSITXSADEPLOY-3054 --- .../persistence/services/AwsS3ObjectStoreFileStorage.java | 1 - .../web/configuration/factory/ObjectStoreSelectorFactory.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index 0647856341..53e813a877 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -27,7 +27,6 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.cglib.core.Local; import org.springframework.http.MediaType; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java index d4a4d27284..ceeb850bf2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java @@ -49,8 +49,8 @@ public ObjectStoreSelectorFactory(String serviceName, EnvironmentServicesFinder this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; this.applicationConfiguration = applicationConfiguration; - this.selectedObjectStore = doSelect(); this.uploadDurationTracker = uploadDurationTracker; + this.selectedObjectStore = doSelect(); } public FileStorage fileStorage() { From 09c663dc92bc6876147dfe12153ca869dce5e2d1 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Mon, 3 Aug 2026 17:58:03 +0300 Subject: [PATCH 4/7] Fix alphabetical order of netty LMCROSSITXSADEPLOY-3054 --- multiapps-controller-persistence/src/main/java/module-info.java | 1 - 1 file changed, 1 deletion(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index c3001f6a66..5924369930 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -81,5 +81,4 @@ requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; - requires io.netty.handler; } From 975cea7c6d7614932c409fb7162a1008573c2250 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Thu, 6 Aug 2026 09:26:52 +0300 Subject: [PATCH 5/7] Adding unit tests for metrics LMCROSSITXSADEPLOY-3054 --- .../monitoring/UploadDurationTrackerTest.java | 190 ++++++++++++++++++ .../monitoring/UploadTimeoutMatcherTest.java | 59 ++++++ .../AppUploaderThreadPoolInformationTest.java | 40 ++++ .../monitoring/UploadDurationMetricsTest.java | 166 +++++++++++++++ 4 files changed, 455 insertions(+) create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java create mode 100644 multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java create mode 100644 multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java new file mode 100644 index 0000000000..f6a1b1905f --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java @@ -0,0 +1,190 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker.UploadPathStatistics; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UploadDurationTrackerTest { + + private UploadDurationTracker uploadDurationTracker; + + @BeforeEach + void setUp() { + uploadDurationTracker = new UploadDurationTracker(); + } + + @Test + void testRecordAppBinaryUploadIncrementsCountAndSum() { + uploadDurationTracker.recordAppBinaryUpload(100, false); + uploadDurationTracker.recordAppBinaryUpload(250, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.totalCount()); + assertEquals(350, statistics.sumDurationMs()); + } + + @Test + void testRecordObjectStoreUploadIncrementsCountAndSum() { + uploadDurationTracker.recordObjectStoreUpload(75, false); + uploadDurationTracker.recordObjectStoreUpload(125, false); + + UploadPathStatistics statistics = uploadDurationTracker.getObjectStoreStatistics(); + assertEquals(2, statistics.totalCount()); + assertEquals(200, statistics.sumDurationMs()); + } + + @Test + void testMaxDurationReturnsLargestAcrossRecords() { + uploadDurationTracker.recordAppBinaryUpload(100, false); + uploadDurationTracker.recordAppBinaryUpload(500, false); + uploadDurationTracker.recordAppBinaryUpload(250, false); + + assertEquals(500, uploadDurationTracker.getAppBinaryStatistics() + .maxDurationMs()); + } + + @Test + void testMaxDurationIsWindowedAndResetsOnRead() { + uploadDurationTracker.recordAppBinaryUpload(400, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(400, statistics.maxDurationMs()); + assertEquals(0, statistics.maxDurationMs()); + } + + @Test + void testMaxDurationStartsFreshWindowAfterRead() { + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + + uploadDurationTracker.recordAppBinaryUpload(400, false); + assertEquals(400, statistics.maxDurationMs()); + + uploadDurationTracker.recordAppBinaryUpload(150, false); + assertEquals(150, statistics.maxDurationMs()); + } + + @Test + void testNegativeDurationIsSetToZeroForSum() { + uploadDurationTracker.recordAppBinaryUpload(100, false); + uploadDurationTracker.recordAppBinaryUpload(-500, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.totalCount()); + assertEquals(100, statistics.sumDurationMs()); + } + + @Test + void testNegativeDurationContributesZeroToMax() { + uploadDurationTracker.recordAppBinaryUpload(-500, false); + + assertEquals(0, uploadDurationTracker.getAppBinaryStatistics() + .maxDurationMs()); + } + + @Test + void testTimeoutIncrementsCumulativeAndWindowedCounts() { + uploadDurationTracker.recordAppBinaryUpload(100, true); + uploadDurationTracker.recordAppBinaryUpload(200, true); + uploadDurationTracker.recordAppBinaryUpload(300, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.timeoutCount()); + assertEquals(2, statistics.timeoutsInWindow()); + } + + @Test + void testTimeoutsInWindowResetsOnReadButTimeoutCountDoesNot() { + uploadDurationTracker.recordAppBinaryUpload(100, true); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(1, statistics.timeoutsInWindow()); + assertEquals(0, statistics.timeoutsInWindow()); + assertEquals(1, statistics.timeoutCount()); + assertEquals(1, statistics.timeoutCount()); + } + + @Test + void testRecordAppBinaryUploadRejectionIncrementsCumulativeAndWindowed() { + uploadDurationTracker.recordAppBinaryUploadRejection(); + uploadDurationTracker.recordAppBinaryUploadRejection(); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.rejectionCount()); + assertEquals(2, statistics.rejectionsInWindow()); + } + + @Test + void testRejectionsInWindowResetsOnReadButRejectionCountDoesNot() { + uploadDurationTracker.recordAppBinaryUploadRejection(); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(1, statistics.rejectionsInWindow()); + assertEquals(0, statistics.rejectionsInWindow()); + assertEquals(1, statistics.rejectionCount()); + assertEquals(1, statistics.rejectionCount()); + } + + @Test + void testAppBinaryAndObjectStoreStatisticsAreIndependent() { + uploadDurationTracker.recordAppBinaryUpload(100, true); + uploadDurationTracker.recordAppBinaryUploadRejection(); + + UploadPathStatistics objectStoreStatistics = uploadDurationTracker.getObjectStoreStatistics(); + assertEquals(0, objectStoreStatistics.totalCount()); + assertEquals(0, objectStoreStatistics.sumDurationMs()); + assertEquals(0, objectStoreStatistics.timeoutCount()); + assertEquals(0, objectStoreStatistics.rejectionCount()); + + uploadDurationTracker.recordObjectStoreUpload(200, false); + + UploadPathStatistics appBinaryStatistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(1, appBinaryStatistics.totalCount()); + assertEquals(1, appBinaryStatistics.totalCount()); + } + + @Test + void testConcurrentRecordsAccumulateSafely() throws InterruptedException { + int threadCount = 100; + long maxDuration = threadCount; + + ExecutorService executor = Executors.newFixedThreadPool(16); + CountDownLatch blockStartLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + + try { + for (int i = 1; i <= threadCount; i++) { + long duration = i; + + executor.submit(() -> { + try { + blockStartLatch.await(); + uploadDurationTracker.recordAppBinaryUpload(duration, false); + } catch (InterruptedException e) { + Thread.currentThread() + .interrupt(); + } finally { + doneLatch.countDown(); + } + }); + } + + blockStartLatch.countDown(); + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + } finally { + executor.shutdownNow(); + } + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(threadCount, statistics.totalCount()); + assertEquals(maxDuration, statistics.maxDurationMs()); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java new file mode 100644 index 0000000000..2442f037f7 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java @@ -0,0 +1,59 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Stream; + +import com.google.cloud.storage.StorageException; +import io.netty.handler.timeout.ReadTimeoutException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UploadTimeoutMatcherTest { + + static Stream testIsUploadTimeoutException() { + return Stream.of( + Arguments.of(new RuntimeException(new SocketTimeoutException("read timed out")), true), + Arguments.of(new RuntimeException(new TimeoutException("operation timed out")), true), + Arguments.of(new RuntimeException(ReadTimeoutException.INSTANCE), true), + Arguments.of(new RuntimeException(new StorageException(504, "gateway timeout")), true), + Arguments.of(new RuntimeException(new StorageException(500, "internal error")), false), + Arguments.of(new RuntimeException(new StorageException(429, "too many requests")), false), + Arguments.of(new RuntimeException(new IllegalStateException("unrelated")), false), + Arguments.of(null, false)); + } + + @ParameterizedTest + @MethodSource + void testIsUploadTimeoutException(Throwable throwable, boolean expected) { + assertEquals(expected, UploadTimeoutMatcher.isUploadTimeoutException(throwable)); + } + + @Test + void testDeeplyNestedTimeoutCauseIsDetected() { + Throwable deepCause = new SocketTimeoutException("read timed out"); + Throwable throwable = new RuntimeException("layer 1", + new IllegalStateException("layer 2", new RuntimeException("layer 3", deepCause))); + + assertTrue(UploadTimeoutMatcher.isUploadTimeoutException(throwable)); + } + + @Test + void testTopLevelTimeoutWithNullCauseIsNotDetected() { + Throwable throwable = new SocketTimeoutException("read timed out"); + + assertFalse(UploadTimeoutMatcher.isUploadTimeoutException(throwable)); + } + + @Test + void testThrowableWithNoCauseIsNotDetected() { + assertFalse(UploadTimeoutMatcher.isUploadTimeoutException(new RuntimeException("no cause"))); + } + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java new file mode 100644 index 0000000000..c9f55efada --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java @@ -0,0 +1,40 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class AppUploaderThreadPoolInformationTest { + + private ThreadPoolExecutor threadPoolExecutor; + private AppUploaderThreadPoolInformation appUploaderThreadPoolInformation; + + @BeforeEach + void setUp() { + threadPoolExecutor = new ThreadPoolExecutor(2, 5, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + appUploaderThreadPoolInformation = new AppUploaderThreadPoolInformation(threadPoolExecutor); + } + + @AfterEach + void cleanUp() { + threadPoolExecutor.shutdownNow(); + } + + @Test + void testGetActiveThreadsReturnsActiveCount() { + assertEquals(threadPoolExecutor.getActiveCount(), appUploaderThreadPoolInformation.getActiveThreads()); + } + + @Test + void testGetMaxThreadsReturnsMaximumPoolSize() { + assertEquals(5, appUploaderThreadPoolInformation.getMaxThreads()); + assertEquals(threadPoolExecutor.getMaximumPoolSize(), appUploaderThreadPoolInformation.getMaxThreads()); + } + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java new file mode 100644 index 0000000000..86207b7ae3 --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java @@ -0,0 +1,166 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker.UploadPathStatistics; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class UploadDurationMetricsTest { + + @Mock + private UploadDurationTracker uploadDurationTracker; + + @Mock + private UploadPathStatistics appBinaryStatistics; + + @Mock + private UploadPathStatistics objectStoreStatistics; + + @Mock + private AppUploaderThreadPoolInformation appUploaderInfo; + + private UploadDurationMetrics uploadDurationMetrics; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + Mockito.when(uploadDurationTracker.getAppBinaryStatistics()) + .thenReturn(appBinaryStatistics); + Mockito.when(uploadDurationTracker.getObjectStoreStatistics()) + .thenReturn(objectStoreStatistics); + uploadDurationMetrics = new UploadDurationMetrics(uploadDurationTracker, appUploaderInfo); + } + + @Test + void testGetAppBinaryUploadTotalCount() { + Mockito.when(appBinaryStatistics.totalCount()) + .thenReturn(11L); + assertEquals(11L, uploadDurationMetrics.getAppBinaryUploadTotalCount()); + Mockito.verify(appBinaryStatistics) + .totalCount(); + } + + @Test + void testGetAppBinaryUploadTimeoutCount() { + Mockito.when(appBinaryStatistics.timeoutCount()) + .thenReturn(12L); + assertEquals(12L, uploadDurationMetrics.getAppBinaryUploadTimeoutCount()); + Mockito.verify(appBinaryStatistics) + .timeoutCount(); + } + + @Test + void testGetAppBinaryUploadMaxDurationMs() { + Mockito.when(appBinaryStatistics.maxDurationMs()) + .thenReturn(13L); + assertEquals(13L, uploadDurationMetrics.getAppBinaryUploadMaxDurationMs()); + Mockito.verify(appBinaryStatistics) + .maxDurationMs(); + } + + @Test + void testGetAppBinaryUploadSumDurationMs() { + Mockito.when(appBinaryStatistics.sumDurationMs()) + .thenReturn(14L); + assertEquals(14L, uploadDurationMetrics.getAppBinaryUploadSumDurationMs()); + Mockito.verify(appBinaryStatistics) + .sumDurationMs(); + } + + @Test + void testGetAppUploaderActiveThreads() { + Mockito.when(appUploaderInfo.getActiveThreads()) + .thenReturn(3); + assertEquals(3, uploadDurationMetrics.getAppUploaderActiveThreads()); + Mockito.verify(appUploaderInfo) + .getActiveThreads(); + } + + @Test + void testGetAppUploaderMaxThreads() { + Mockito.when(appUploaderInfo.getMaxThreads()) + .thenReturn(7); + assertEquals(7, uploadDurationMetrics.getAppUploaderMaxThreads()); + Mockito.verify(appUploaderInfo) + .getMaxThreads(); + } + + @Test + void testGetAppUploaderRejectionCount() { + Mockito.when(appBinaryStatistics.rejectionCount()) + .thenReturn(15L); + assertEquals(15L, uploadDurationMetrics.getAppUploaderRejectionCount()); + Mockito.verify(appBinaryStatistics) + .rejectionCount(); + } + + @Test + void testGetAppUploaderRejectionCountInWindow() { + Mockito.when(appBinaryStatistics.rejectionsInWindow()) + .thenReturn(16L); + assertEquals(16L, uploadDurationMetrics.getAppUploaderRejectionCountInWindow()); + Mockito.verify(appBinaryStatistics) + .rejectionsInWindow(); + } + + @Test + void testGetObjectStoreUploadTotalCount() { + Mockito.when(objectStoreStatistics.totalCount()) + .thenReturn(21L); + assertEquals(21L, uploadDurationMetrics.getObjectStoreUploadTotalCount()); + Mockito.verify(objectStoreStatistics) + .totalCount(); + } + + @Test + void testGetObjectStoreUploadTimeoutCount() { + Mockito.when(objectStoreStatistics.timeoutCount()) + .thenReturn(22L); + assertEquals(22L, uploadDurationMetrics.getObjectStoreUploadTimeoutCount()); + Mockito.verify(objectStoreStatistics) + .timeoutCount(); + } + + @Test + void testGetObjectStoreUploadMaxDurationMs() { + Mockito.when(objectStoreStatistics.maxDurationMs()) + .thenReturn(23L); + assertEquals(23L, uploadDurationMetrics.getObjectStoreUploadMaxDurationMs()); + Mockito.verify(objectStoreStatistics) + .maxDurationMs(); + } + + @Test + void testGetObjectStoreUploadSumDurationMs() { + Mockito.when(objectStoreStatistics.sumDurationMs()) + .thenReturn(24L); + assertEquals(24L, uploadDurationMetrics.getObjectStoreUploadSumDurationMs()); + Mockito.verify(objectStoreStatistics) + .sumDurationMs(); + } + + @Test + void testGetAppBinaryUploadTimeoutsInWindow() { + Mockito.when(appBinaryStatistics.timeoutsInWindow()) + .thenReturn(17L); + assertEquals(17L, uploadDurationMetrics.getAppBinaryUploadTimeoutsInWindow()); + Mockito.verify(appBinaryStatistics) + .timeoutsInWindow(); + } + + @Test + void testGetObjectStoreUploadTimeoutsInWindow() { + Mockito.when(objectStoreStatistics.timeoutsInWindow()) + .thenReturn(25L); + assertEquals(25L, uploadDurationMetrics.getObjectStoreUploadTimeoutsInWindow()); + Mockito.verify(objectStoreStatistics) + .timeoutsInWindow(); + } + +} From e92d064ad5193055042684a81fc5a42d1ab83219 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Thu, 6 Aug 2026 12:11:24 +0300 Subject: [PATCH 6/7] Fixing time zone sensitive upload duration LMCROSSITXSADEPLOY-3054 --- .../services/AwsS3ObjectStoreFileStorage.java | 7 ++++--- .../services/AzureObjectStoreFileStorage.java | 7 ++++--- .../services/GcpObjectStoreFileStorage.java | 7 ++++--- .../services/JCloudsObjectStoreFileStorage.java | 7 ++++--- .../process/steps/UploadAppAsyncExecution.java | 12 ++++++------ .../controller/process/variables/Variables.java | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index 53e813a877..f9e43535fe 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -5,6 +5,7 @@ import java.net.URI; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -109,7 +110,7 @@ protected ClientOverrideConfiguration buildClientOverrideConfig() { @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { - LocalDateTime startTime = LocalDateTime.now(); + Instant startTime = Instant.now(); long fileSize = fileEntry.getSize() .longValue(); PutObjectRequest request = PutObjectRequest.builder() @@ -331,8 +332,8 @@ public void destroy() { s3Client.close(); } - private long getElapsedTimeInMillis(LocalDateTime startTime) { - return Duration.between(startTime, LocalDateTime.now()) + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) .toMillis(); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index 6bc2799a2d..ee3f7945a7 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -6,6 +6,7 @@ import java.net.URL; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -60,7 +61,7 @@ public AzureObjectStoreFileStorage(Map credentials, UploadDurati @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); - LocalDateTime startTime = LocalDateTime.now(); + Instant startTime = Instant.now(); try { ParallelTransferOptions pto = new ParallelTransferOptions().setMaxSingleUploadSizeLong(MAX_SINGLE_UPLOAD_SIZE) .setMaxConcurrency(MAX_CONCURRENCY) @@ -228,8 +229,8 @@ private int removeBlobsByFilter(Predicate filter) { return deletedBlobsResult; } - private long getElapsedTimeInMillis(LocalDateTime startTime) { - return Duration.between(startTime, LocalDateTime.now()) + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) .toMillis(); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index c4d73c8711..d8c87b7824 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -28,6 +28,7 @@ import java.nio.channels.Channels; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -97,7 +98,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { - LocalDateTime startTime = LocalDateTime.now(); + Instant startTime = Instant.now(); try { storage.createFrom(blobInfo, content); } catch (IOException | StorageException e) { @@ -294,8 +295,8 @@ private static final class CredentialKeys { static final String BUCKET = "bucket"; } - private long getElapsedTimeInMillis(LocalDateTime startTime) { - return Duration.between(startTime, LocalDateTime.now()) + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) .toMillis(); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index e7bea41567..4b9d196dc0 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.HashSet; import java.util.List; @@ -61,7 +62,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentLength(fileSize) .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); - LocalDateTime startTime = LocalDateTime.now(); + Instant startTime = Instant.now(); try { putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); @@ -272,8 +273,8 @@ private Set getAllEntries(ListContainerOptions options) { return entries; } - private long getElapsedTimeInMillis(LocalDateTime startTime) { - return Duration.between(startTime, LocalDateTime.now()) + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) .toMillis(); } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java index 14f13392c0..72402c3f04 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.text.MessageFormat; import java.time.Duration; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Set; @@ -72,7 +72,7 @@ public AsyncExecutionState execute(ProcessContext context) { return AsyncExecutionState.FINISHED; } if (context.getVariable(Variables.UPLOAD_START_TIME) == null) { - context.setVariable(Variables.UPLOAD_START_TIME, LocalDateTime.now()); + context.setVariable(Variables.UPLOAD_START_TIME, Instant.now()); } ApplicationToUploadContext applicationToUploadContext = buildApplicationToUploadContext(context, applicationToProcess); CloudControllerClient client = context.getControllerClient(); @@ -153,10 +153,10 @@ private CloudPackage asyncUploadFiles(CloudControllerClient client, ApplicationT } private Path extractApplicationFromArchive(ApplicationToUploadContext applicationToUploadContext) { - LocalDateTime startTime = LocalDateTime.now(); + Instant startTime = Instant.now(); Path extractedAppPath = extractFromMtar( createApplicationArchiveContext(applicationToUploadContext, applicationConfiguration.getMaxResourceFileSize())); - long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) + long timeElapsedForUpload = Duration.between(startTime, Instant.now()) .toMillis(); applicationToUploadContext.getStepLogger() .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_APP_BINARY_DOWNLOAD_0_IN_MILLIS, @@ -215,8 +215,8 @@ private void attemptToUpdateApplicationDigest(CloudControllerClient client, Clou } private long getElapsedTimeInMillis(ProcessContext context) { - LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); - return Duration.between(startTime, LocalDateTime.now()) + Instant startTime = context.getVariable(Variables.UPLOAD_START_TIME); + return Duration.between(startTime, Instant.now()) .toMillis(); } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java index e498355147..efe9f0f026 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java @@ -683,7 +683,7 @@ public interface Variables { .name("waitBindServiceTimeout") .defaultValue(Duration.ofHours(1)) .build(); - Variable UPLOAD_START_TIME = ImmutableSimpleVariable. builder() + Variable UPLOAD_START_TIME = ImmutableSimpleVariable. builder() .name("uploadStartTime") .build(); From e1320efb16a4a5713f73617a85f33b992ead5b5b Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Thu, 6 Aug 2026 13:40:29 +0300 Subject: [PATCH 7/7] Fixing code smells LMCROSSITXSADEPLOY-3054 --- .../monitoring/UploadDurationMetricsTest.java | 123 +++++++++--------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java index 86207b7ae3..6826959f7a 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java @@ -5,10 +5,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class UploadDurationMetricsTest { @@ -30,137 +31,137 @@ class UploadDurationMetricsTest { void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - Mockito.when(uploadDurationTracker.getAppBinaryStatistics()) - .thenReturn(appBinaryStatistics); - Mockito.when(uploadDurationTracker.getObjectStoreStatistics()) - .thenReturn(objectStoreStatistics); + when(uploadDurationTracker.getAppBinaryStatistics()) + .thenReturn(appBinaryStatistics); + when(uploadDurationTracker.getObjectStoreStatistics()) + .thenReturn(objectStoreStatistics); uploadDurationMetrics = new UploadDurationMetrics(uploadDurationTracker, appUploaderInfo); } @Test void testGetAppBinaryUploadTotalCount() { - Mockito.when(appBinaryStatistics.totalCount()) - .thenReturn(11L); + when(appBinaryStatistics.totalCount()) + .thenReturn(11L); assertEquals(11L, uploadDurationMetrics.getAppBinaryUploadTotalCount()); - Mockito.verify(appBinaryStatistics) - .totalCount(); + verify(appBinaryStatistics) + .totalCount(); } @Test void testGetAppBinaryUploadTimeoutCount() { - Mockito.when(appBinaryStatistics.timeoutCount()) - .thenReturn(12L); + when(appBinaryStatistics.timeoutCount()) + .thenReturn(12L); assertEquals(12L, uploadDurationMetrics.getAppBinaryUploadTimeoutCount()); - Mockito.verify(appBinaryStatistics) - .timeoutCount(); + verify(appBinaryStatistics) + .timeoutCount(); } @Test void testGetAppBinaryUploadMaxDurationMs() { - Mockito.when(appBinaryStatistics.maxDurationMs()) - .thenReturn(13L); + when(appBinaryStatistics.maxDurationMs()) + .thenReturn(13L); assertEquals(13L, uploadDurationMetrics.getAppBinaryUploadMaxDurationMs()); - Mockito.verify(appBinaryStatistics) - .maxDurationMs(); + verify(appBinaryStatistics) + .maxDurationMs(); } @Test void testGetAppBinaryUploadSumDurationMs() { - Mockito.when(appBinaryStatistics.sumDurationMs()) - .thenReturn(14L); + when(appBinaryStatistics.sumDurationMs()) + .thenReturn(14L); assertEquals(14L, uploadDurationMetrics.getAppBinaryUploadSumDurationMs()); - Mockito.verify(appBinaryStatistics) - .sumDurationMs(); + verify(appBinaryStatistics) + .sumDurationMs(); } @Test void testGetAppUploaderActiveThreads() { - Mockito.when(appUploaderInfo.getActiveThreads()) - .thenReturn(3); + when(appUploaderInfo.getActiveThreads()) + .thenReturn(3); assertEquals(3, uploadDurationMetrics.getAppUploaderActiveThreads()); - Mockito.verify(appUploaderInfo) - .getActiveThreads(); + verify(appUploaderInfo) + .getActiveThreads(); } @Test void testGetAppUploaderMaxThreads() { - Mockito.when(appUploaderInfo.getMaxThreads()) - .thenReturn(7); + when(appUploaderInfo.getMaxThreads()) + .thenReturn(7); assertEquals(7, uploadDurationMetrics.getAppUploaderMaxThreads()); - Mockito.verify(appUploaderInfo) - .getMaxThreads(); + verify(appUploaderInfo) + .getMaxThreads(); } @Test void testGetAppUploaderRejectionCount() { - Mockito.when(appBinaryStatistics.rejectionCount()) - .thenReturn(15L); + when(appBinaryStatistics.rejectionCount()) + .thenReturn(15L); assertEquals(15L, uploadDurationMetrics.getAppUploaderRejectionCount()); - Mockito.verify(appBinaryStatistics) - .rejectionCount(); + verify(appBinaryStatistics) + .rejectionCount(); } @Test void testGetAppUploaderRejectionCountInWindow() { - Mockito.when(appBinaryStatistics.rejectionsInWindow()) - .thenReturn(16L); + when(appBinaryStatistics.rejectionsInWindow()) + .thenReturn(16L); assertEquals(16L, uploadDurationMetrics.getAppUploaderRejectionCountInWindow()); - Mockito.verify(appBinaryStatistics) - .rejectionsInWindow(); + verify(appBinaryStatistics) + .rejectionsInWindow(); } @Test void testGetObjectStoreUploadTotalCount() { - Mockito.when(objectStoreStatistics.totalCount()) - .thenReturn(21L); + when(objectStoreStatistics.totalCount()) + .thenReturn(21L); assertEquals(21L, uploadDurationMetrics.getObjectStoreUploadTotalCount()); - Mockito.verify(objectStoreStatistics) - .totalCount(); + verify(objectStoreStatistics) + .totalCount(); } @Test void testGetObjectStoreUploadTimeoutCount() { - Mockito.when(objectStoreStatistics.timeoutCount()) - .thenReturn(22L); + when(objectStoreStatistics.timeoutCount()) + .thenReturn(22L); assertEquals(22L, uploadDurationMetrics.getObjectStoreUploadTimeoutCount()); - Mockito.verify(objectStoreStatistics) - .timeoutCount(); + verify(objectStoreStatistics) + .timeoutCount(); } @Test void testGetObjectStoreUploadMaxDurationMs() { - Mockito.when(objectStoreStatistics.maxDurationMs()) - .thenReturn(23L); + when(objectStoreStatistics.maxDurationMs()) + .thenReturn(23L); assertEquals(23L, uploadDurationMetrics.getObjectStoreUploadMaxDurationMs()); - Mockito.verify(objectStoreStatistics) - .maxDurationMs(); + verify(objectStoreStatistics) + .maxDurationMs(); } @Test void testGetObjectStoreUploadSumDurationMs() { - Mockito.when(objectStoreStatistics.sumDurationMs()) - .thenReturn(24L); + when(objectStoreStatistics.sumDurationMs()) + .thenReturn(24L); assertEquals(24L, uploadDurationMetrics.getObjectStoreUploadSumDurationMs()); - Mockito.verify(objectStoreStatistics) - .sumDurationMs(); + verify(objectStoreStatistics) + .sumDurationMs(); } @Test void testGetAppBinaryUploadTimeoutsInWindow() { - Mockito.when(appBinaryStatistics.timeoutsInWindow()) - .thenReturn(17L); + when(appBinaryStatistics.timeoutsInWindow()) + .thenReturn(17L); assertEquals(17L, uploadDurationMetrics.getAppBinaryUploadTimeoutsInWindow()); - Mockito.verify(appBinaryStatistics) - .timeoutsInWindow(); + verify(appBinaryStatistics) + .timeoutsInWindow(); } @Test void testGetObjectStoreUploadTimeoutsInWindow() { - Mockito.when(objectStoreStatistics.timeoutsInWindow()) - .thenReturn(25L); + when(objectStoreStatistics.timeoutsInWindow()) + .thenReturn(25L); assertEquals(25L, uploadDurationMetrics.getObjectStoreUploadTimeoutsInWindow()); - Mockito.verify(objectStoreStatistics) - .timeoutsInWindow(); + verify(objectStoreStatistics) + .timeoutsInWindow(); } }