diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index b5078c3fc3..5924369930 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -9,6 +9,7 @@ exports org.cloudfoundry.multiapps.controller.persistence.model; exports org.cloudfoundry.multiapps.controller.persistence.model.adapter; exports org.cloudfoundry.multiapps.controller.persistence.model.filters; + exports org.cloudfoundry.multiapps.controller.persistence.monitoring; exports org.cloudfoundry.multiapps.controller.persistence.query; exports org.cloudfoundry.multiapps.controller.persistence.query.criteria; exports org.cloudfoundry.multiapps.controller.persistence.query.impl; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 3e44634d02..459531195f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -74,6 +74,10 @@ public final class Messages { // INFO log messages: public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}"; + public static final String TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for JClouds object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for GCP object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for Azure object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for AWS object store upload: {0} in millis"; // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java new file mode 100644 index 0000000000..d6ac887b8b --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java @@ -0,0 +1,99 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import jakarta.inject.Named; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +@Named +public class UploadDurationTracker { + + private final UploadPathStatistics appBinaryStatistics = new UploadPathStatistics(); + + private final UploadPathStatistics objectStoreStatistics = new UploadPathStatistics(); + + public void recordAppBinaryUpload(long durationMillis, boolean timedOut) { + appBinaryStatistics.record(durationMillis, timedOut); + } + + public void recordObjectStoreUpload(long durationMillis, boolean timedOut) { + objectStoreStatistics.record(durationMillis, timedOut); + } + + public void recordAppBinaryUploadRejection() { + appBinaryStatistics.recordRejection(); + } + + public UploadPathStatistics getAppBinaryStatistics() { + return this.appBinaryStatistics; + } + + public UploadPathStatistics getObjectStoreStatistics() { + return this.objectStoreStatistics; + } + + public static final class UploadPathStatistics { + + private final LongAdder total = new LongAdder(); + + private final LongAdder timeouts = new LongAdder(); + + private final LongAdder sumDuration = new LongAdder(); + + private final LongAdder rejections = new LongAdder(); + + private final AtomicLong rejectionsInWindow = new AtomicLong(0); + + private final AtomicLong maxDuration = new AtomicLong(0); + + private final AtomicLong timeoutsInWindow = new AtomicLong(0); + + public void record(long durationMillis, boolean timedOut) { + long duration = Math.max(0, durationMillis); + total.increment(); + + if (timedOut) { + timeouts.increment(); + timeoutsInWindow.incrementAndGet(); + } + + sumDuration.add(duration); + maxDuration.accumulateAndGet(duration, Math::max); + } + + public void recordRejection() { + rejections.increment(); + rejectionsInWindow.incrementAndGet(); + } + + public long totalCount() { + return total.sum(); + } + + public long timeoutCount() { + return timeouts.sum(); + } + + public long maxDurationMs() { + return maxDuration.getAndSet(0); + } + + public long sumDurationMs() { + return sumDuration.sum(); + } + + public long timeoutsInWindow() { + return timeoutsInWindow.getAndSet(0); + } + + public long rejectionCount() { + return rejections.sum(); + } + + public long rejectionsInWindow() { + return rejectionsInWindow.getAndSet(0); + } + + } + +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java new file mode 100644 index 0000000000..f229a53aee --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java @@ -0,0 +1,31 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import com.google.cloud.storage.StorageException; +import io.netty.handler.timeout.ReadTimeoutException; + +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; + +public class UploadTimeoutMatcher { + + private UploadTimeoutMatcher() { + + } + + public static boolean isUploadTimeoutException(Throwable throwable) { + if (throwable == null) { + return false; + } + + Throwable cause = throwable.getCause(); + while (cause != null) { + if (cause instanceof SocketTimeoutException || cause instanceof TimeoutException || cause instanceof ReadTimeoutException || ( + cause instanceof StorageException + && ((StorageException) cause).getCode() == 504)) { + return true; + } + cause = cause.getCause(); + } + return false; + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index a075038c51..f9e43535fe 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -4,6 +4,8 @@ import java.io.InputStream; import java.net.URI; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -16,8 +18,11 @@ import java.util.concurrent.Executors; import java.util.function.BiPredicate; import java.util.function.Consumer; + import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -57,10 +62,12 @@ public class AwsS3ObjectStoreFileStorage extends ObjectStoreFileStorage { private final S3Client s3Client; private final String bucketName; + private final UploadDurationTracker uploadDurationTracker; - public AwsS3ObjectStoreFileStorage(Map credentials) { + public AwsS3ObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.s3Client = createS3Client(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected S3Client createS3Client(Map credentials) { @@ -103,6 +110,7 @@ protected ClientOverrideConfiguration buildClientOverrideConfig() { @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + Instant startTime = Instant.now(); long fileSize = fileEntry.getSize() .longValue(); PutObjectRequest request = PutObjectRequest.builder() @@ -115,8 +123,12 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage try { s3Client.putObject(request, RequestBody.fromInputStream(new BufferedInputStream(content), fileSize)); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } catch (Exception e) { LOGGER.error(MessageFormat.format(Messages.S3_UPLOAD_FAILED_FILE_0_SIZE_1, fileEntry.getName(), fileSize, e)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED, fileEntry.getName(), fileEntry.getNamespace()), e); } @@ -183,14 +195,15 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); + int deletedFiles = deleteByFilterAndCount( + (key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterByModificationTime(metadata, key, - modificationTime)); + modificationTime)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_MODIFIED_BEFORE_1, deletedFiles, modificationTime)); return deletedFiles; } @@ -319,6 +332,11 @@ public void destroy() { s3Client.close(); } + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) + .toMillis(); + } + private static final class CredentialKeys { static final String ACCESS_KEY_ID = "access_key_id"; static final String SECRET_ACCESS_KEY = "secret_access_key"; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index 721e7234b8..ee3f7945a7 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -5,6 +5,8 @@ import java.net.MalformedURLException; import java.net.URL; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -29,6 +31,8 @@ import com.azure.storage.blob.options.BlobParallelUploadOptions; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -46,15 +50,18 @@ public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { private static final int MAX_CONCURRENCY = 5; private final HttpClient httpClient; private final BlobContainerClient containerClient; + private final UploadDurationTracker uploadDurationTracker; - public AzureObjectStoreFileStorage(Map credentials) { + public AzureObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.httpClient = new JdkHttpClientBuilder().build(); this.containerClient = createContainerClient(credentials); + this.uploadDurationTracker = uploadDurationTracker; } @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + Instant startTime = Instant.now(); try { ParallelTransferOptions pto = new ParallelTransferOptions().setMaxSingleUploadSizeLong(MAX_SINGLE_UPLOAD_SIZE) .setMaxConcurrency(MAX_CONCURRENCY) @@ -67,10 +74,14 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage blobClient.uploadWithResponse(blobParallelUploadOptions, ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, null); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileEntry.getSize() - .longValue())); + .longValue())); } catch (BlobStorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -218,6 +229,11 @@ private int removeBlobsByFilter(Predicate filter) { return deletedBlobsResult; } + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) + .toMillis(); + } + protected Set getEntryNames(Predicate filter) { BlobListDetails blobListDetails = new BlobListDetails().setRetrieveMetadata(true); ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(blobListDetails); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 2a0757eb5f..d8c87b7824 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -13,6 +13,8 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -25,6 +27,8 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -41,10 +45,12 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; + private final UploadDurationTracker uploadDurationTracker; - public GcpObjectStoreFileStorage(Map credentials) { + public GcpObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.storage = createObjectStoreStorage(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected Storage createObjectStoreStorage(Map credentials) { @@ -92,11 +98,16 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { + Instant startTime = Instant.now(); try { storage.createFrom(blobInfo, content); } catch (IOException | StorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -283,4 +294,10 @@ private static final class CredentialKeys { static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; static final String BUCKET = "bucket"; } + + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 287aa5a344..4b9d196dc0 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -3,6 +3,8 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.HashSet; import java.util.List; @@ -10,9 +12,12 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; + import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.jclouds.blobstore.BlobStore; @@ -37,10 +42,12 @@ public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage { private final BlobStore blobStore; private final String container; + private final UploadDurationTracker uploadDurationTracker; - public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { + public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container, UploadDurationTracker uploadDurationTracker) { this.blobStore = blobStore; this.container = container; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -55,13 +62,18 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentLength(fileSize) .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); + Instant startTime = Instant.now(); try { putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), fileEntry.getNamespace())); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -95,7 +107,7 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { int deletedFiles = removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, - namespace)); + namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @@ -260,4 +272,10 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } + + private long getElapsedTimeInMillis(Instant startTime) { + return Duration.between(startTime, Instant.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java new file mode 100644 index 0000000000..f6a1b1905f --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTrackerTest.java @@ -0,0 +1,190 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker.UploadPathStatistics; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UploadDurationTrackerTest { + + private UploadDurationTracker uploadDurationTracker; + + @BeforeEach + void setUp() { + uploadDurationTracker = new UploadDurationTracker(); + } + + @Test + void testRecordAppBinaryUploadIncrementsCountAndSum() { + uploadDurationTracker.recordAppBinaryUpload(100, false); + uploadDurationTracker.recordAppBinaryUpload(250, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.totalCount()); + assertEquals(350, statistics.sumDurationMs()); + } + + @Test + void testRecordObjectStoreUploadIncrementsCountAndSum() { + uploadDurationTracker.recordObjectStoreUpload(75, false); + uploadDurationTracker.recordObjectStoreUpload(125, false); + + UploadPathStatistics statistics = uploadDurationTracker.getObjectStoreStatistics(); + assertEquals(2, statistics.totalCount()); + assertEquals(200, statistics.sumDurationMs()); + } + + @Test + void testMaxDurationReturnsLargestAcrossRecords() { + uploadDurationTracker.recordAppBinaryUpload(100, false); + uploadDurationTracker.recordAppBinaryUpload(500, false); + uploadDurationTracker.recordAppBinaryUpload(250, false); + + assertEquals(500, uploadDurationTracker.getAppBinaryStatistics() + .maxDurationMs()); + } + + @Test + void testMaxDurationIsWindowedAndResetsOnRead() { + uploadDurationTracker.recordAppBinaryUpload(400, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(400, statistics.maxDurationMs()); + assertEquals(0, statistics.maxDurationMs()); + } + + @Test + void testMaxDurationStartsFreshWindowAfterRead() { + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + + uploadDurationTracker.recordAppBinaryUpload(400, false); + assertEquals(400, statistics.maxDurationMs()); + + uploadDurationTracker.recordAppBinaryUpload(150, false); + assertEquals(150, statistics.maxDurationMs()); + } + + @Test + void testNegativeDurationIsSetToZeroForSum() { + uploadDurationTracker.recordAppBinaryUpload(100, false); + uploadDurationTracker.recordAppBinaryUpload(-500, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.totalCount()); + assertEquals(100, statistics.sumDurationMs()); + } + + @Test + void testNegativeDurationContributesZeroToMax() { + uploadDurationTracker.recordAppBinaryUpload(-500, false); + + assertEquals(0, uploadDurationTracker.getAppBinaryStatistics() + .maxDurationMs()); + } + + @Test + void testTimeoutIncrementsCumulativeAndWindowedCounts() { + uploadDurationTracker.recordAppBinaryUpload(100, true); + uploadDurationTracker.recordAppBinaryUpload(200, true); + uploadDurationTracker.recordAppBinaryUpload(300, false); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.timeoutCount()); + assertEquals(2, statistics.timeoutsInWindow()); + } + + @Test + void testTimeoutsInWindowResetsOnReadButTimeoutCountDoesNot() { + uploadDurationTracker.recordAppBinaryUpload(100, true); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(1, statistics.timeoutsInWindow()); + assertEquals(0, statistics.timeoutsInWindow()); + assertEquals(1, statistics.timeoutCount()); + assertEquals(1, statistics.timeoutCount()); + } + + @Test + void testRecordAppBinaryUploadRejectionIncrementsCumulativeAndWindowed() { + uploadDurationTracker.recordAppBinaryUploadRejection(); + uploadDurationTracker.recordAppBinaryUploadRejection(); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(2, statistics.rejectionCount()); + assertEquals(2, statistics.rejectionsInWindow()); + } + + @Test + void testRejectionsInWindowResetsOnReadButRejectionCountDoesNot() { + uploadDurationTracker.recordAppBinaryUploadRejection(); + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(1, statistics.rejectionsInWindow()); + assertEquals(0, statistics.rejectionsInWindow()); + assertEquals(1, statistics.rejectionCount()); + assertEquals(1, statistics.rejectionCount()); + } + + @Test + void testAppBinaryAndObjectStoreStatisticsAreIndependent() { + uploadDurationTracker.recordAppBinaryUpload(100, true); + uploadDurationTracker.recordAppBinaryUploadRejection(); + + UploadPathStatistics objectStoreStatistics = uploadDurationTracker.getObjectStoreStatistics(); + assertEquals(0, objectStoreStatistics.totalCount()); + assertEquals(0, objectStoreStatistics.sumDurationMs()); + assertEquals(0, objectStoreStatistics.timeoutCount()); + assertEquals(0, objectStoreStatistics.rejectionCount()); + + uploadDurationTracker.recordObjectStoreUpload(200, false); + + UploadPathStatistics appBinaryStatistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(1, appBinaryStatistics.totalCount()); + assertEquals(1, appBinaryStatistics.totalCount()); + } + + @Test + void testConcurrentRecordsAccumulateSafely() throws InterruptedException { + int threadCount = 100; + long maxDuration = threadCount; + + ExecutorService executor = Executors.newFixedThreadPool(16); + CountDownLatch blockStartLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + + try { + for (int i = 1; i <= threadCount; i++) { + long duration = i; + + executor.submit(() -> { + try { + blockStartLatch.await(); + uploadDurationTracker.recordAppBinaryUpload(duration, false); + } catch (InterruptedException e) { + Thread.currentThread() + .interrupt(); + } finally { + doneLatch.countDown(); + } + }); + } + + blockStartLatch.countDown(); + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + } finally { + executor.shutdownNow(); + } + + UploadPathStatistics statistics = uploadDurationTracker.getAppBinaryStatistics(); + assertEquals(threadCount, statistics.totalCount()); + assertEquals(maxDuration, statistics.maxDurationMs()); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java new file mode 100644 index 0000000000..2442f037f7 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcherTest.java @@ -0,0 +1,59 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Stream; + +import com.google.cloud.storage.StorageException; +import io.netty.handler.timeout.ReadTimeoutException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UploadTimeoutMatcherTest { + + static Stream testIsUploadTimeoutException() { + return Stream.of( + Arguments.of(new RuntimeException(new SocketTimeoutException("read timed out")), true), + Arguments.of(new RuntimeException(new TimeoutException("operation timed out")), true), + Arguments.of(new RuntimeException(ReadTimeoutException.INSTANCE), true), + Arguments.of(new RuntimeException(new StorageException(504, "gateway timeout")), true), + Arguments.of(new RuntimeException(new StorageException(500, "internal error")), false), + Arguments.of(new RuntimeException(new StorageException(429, "too many requests")), false), + Arguments.of(new RuntimeException(new IllegalStateException("unrelated")), false), + Arguments.of(null, false)); + } + + @ParameterizedTest + @MethodSource + void testIsUploadTimeoutException(Throwable throwable, boolean expected) { + assertEquals(expected, UploadTimeoutMatcher.isUploadTimeoutException(throwable)); + } + + @Test + void testDeeplyNestedTimeoutCauseIsDetected() { + Throwable deepCause = new SocketTimeoutException("read timed out"); + Throwable throwable = new RuntimeException("layer 1", + new IllegalStateException("layer 2", new RuntimeException("layer 3", deepCause))); + + assertTrue(UploadTimeoutMatcher.isUploadTimeoutException(throwable)); + } + + @Test + void testTopLevelTimeoutWithNullCauseIsNotDetected() { + Throwable throwable = new SocketTimeoutException("read timed out"); + + assertFalse(UploadTimeoutMatcher.isUploadTimeoutException(throwable)); + } + + @Test + void testThrowableWithNoCauseIsNotDetected() { + assertFalse(UploadTimeoutMatcher.isUploadTimeoutException(new RuntimeException("no cause"))); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java index b5b5a08011..e41c95d561 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java @@ -2,6 +2,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -62,16 +63,19 @@ class AwsS3ObjectStoreFileStorageTest { @Mock private FileContentProcessor fileContentProcessor; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AwsS3ObjectStoreFileStorage fileStorage; private final InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private static final String TEST_SPACE_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_SPACE_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String NAMESPACE = "namespace"; private static final String NAMESPACE_2 = "namespace_2"; @@ -80,7 +84,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME)) { + fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME), uploadDurationTracker) { @Override protected S3Client createS3Client(Map credentials) { @@ -136,7 +140,7 @@ void testGetFileEntriesWithoutContentWithoutMatches() { } @Test - void testExistsInObjectStoreWhenFileExists() { + void testExistsInObjectStoreWhenFileExists() { when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() .build()); FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); @@ -232,7 +236,7 @@ void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { @Test void testDeleteFilesModifiedBefore() { LocalDateTime oldModified = FILE_TIMESTAMP - .minusMinutes(15); + .minusMinutes(15); setupListObjectsWithKeys(TEST_ID, TEST_ID_2); setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, oldModified); @@ -250,7 +254,7 @@ void testDeleteFilesModifiedBeforeWithNoOldFiles() { setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, FILE_TIMESTAMP); LocalDateTime cutoff = FILE_TIMESTAMP - .minusDays(1); + .minusDays(1); int deletedCount = fileStorage.deleteFilesModifiedBefore(cutoff); assertEquals(0, deletedCount); @@ -532,7 +536,7 @@ void testBuildClientOverrideConfigSetsApiCallAttemptTimeout() { assertTrue(config.apiCallAttemptTimeout() .isPresent()); assertEquals(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallAttemptTimeout() - .get()); + .get()); } @Test @@ -544,8 +548,8 @@ void testBuildClientOverrideConfigSetsRetryStrategy() { assertInstanceOf(StandardRetryStrategy.class, config.retryStrategy() .get()); assertEquals(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG, config.retryStrategy() - .get() - .maxAttempts()); + .get() + .maxAttempts()); } @Test @@ -555,7 +559,7 @@ void testCreateS3ClientWithValidCredentials() { "bucket", "test-bucket", "host", "s3.amazonaws.com", "region", "eu-central-1"); - AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials); + AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials, uploadDurationTracker); assertNotNull(storage); storage.destroy(); diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java index 437981fa8a..adb3361faa 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -20,6 +20,7 @@ import com.azure.storage.blob.specialized.BlobInputStream; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -51,6 +52,9 @@ class AzureObjectStoreFileStorageTest { @Mock private BlobInputStream blobInputStream; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AzureObjectStoreFileStorage fileStorage; private InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private final String TEST_SPACE_ID = UUID.randomUUID() @@ -70,7 +74,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AzureObjectStoreFileStorage(Map.of()) { + fileStorage = new AzureObjectStoreFileStorage(Map.of(), uploadDurationTracker) { @Override protected BlobContainerClient createContainerClient(Map credentials) { diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 14264ffa72..6fb0f3cf44 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -8,6 +8,7 @@ import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,13 +37,14 @@ class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; private Storage mockedStorage; private GcpObjectStoreFileStorage mockedGcpFileStorage; + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @Override @BeforeEach public void setUp() { storage = LocalStorageHelper.getOptions() .getService(); - fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { @@ -54,7 +56,7 @@ protected Storage createObjectStoreStorage(Map credentials) { namespace = UUID.randomUUID() .toString(); mockedStorage = mock(Storage.class); - mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { return mockedStorage; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 896cf90ee1..77de41d591 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -16,10 +16,12 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; + import jakarta.xml.bind.DatatypeConverter; import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; @@ -28,6 +30,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; + import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -57,10 +60,12 @@ class JCloudsObjectStoreFileStorageTest { private BlobStoreContext blobStoreContext; + private UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); + @BeforeEach protected void setUp() { createBlobStoreContext(); - fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER, uploadDurationTracker) { @Override protected long getRetryWaitTime() { return 1; @@ -233,7 +238,7 @@ void testConnection() { @Test void testTestConnectionWhenContainerExists() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(true); @@ -244,7 +249,7 @@ void testTestConnectionWhenContainerExists() { @Test void testTestConnectionWhenContainerDoesNotExist() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(false); @@ -404,7 +409,7 @@ protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(true); @@ -418,7 +423,7 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { void existsInObjectStoreWithMockedBlobStoreFileNotFound() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(false); @@ -433,7 +438,8 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesCorrectContainer() { BlobStore mockBlobStore = mock(BlobStore.class); String testContainer = "test-container"; FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer, + uploadDurationTracker); when(mockBlobStore.blobExists(testContainer, fileEntry.getId())).thenReturn(true); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java index 7f83745815..72402c3f04 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.text.MessageFormat; import java.time.Duration; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Set; @@ -20,12 +20,14 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; import org.cloudfoundry.multiapps.controller.client.lib.domain.UploadStatusCallbackExtended; import org.cloudfoundry.multiapps.controller.core.Constants; +import org.cloudfoundry.multiapps.controller.core.cloudlogging.OperationLogsExporter; import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationEnvironmentUpdater; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration; -import org.cloudfoundry.multiapps.controller.core.cloudlogging.OperationLogsExporter; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.services.ProcessLoggerPersister; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.context.ApplicationToUploadContext; @@ -48,15 +50,17 @@ public class UploadAppAsyncExecution implements AsyncExecution { private final ApplicationConfiguration applicationConfiguration; private final ExecutorService appUploaderThreadPool; private final OperationLogsExporter operationLogsExporter; + private final UploadDurationTracker uploadDurationTracker; public UploadAppAsyncExecution(ApplicationZipBuilder applicationZipBuilder, ProcessLoggerPersister processLoggerPersister, ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool, - OperationLogsExporter operationLogsExporter) { + OperationLogsExporter operationLogsExporter, UploadDurationTracker uploadDurationTracker) { this.applicationZipBuilder = applicationZipBuilder; this.processLoggerPersister = processLoggerPersister; this.applicationConfiguration = applicationConfiguration; this.appUploaderThreadPool = appUploaderThreadPool; this.operationLogsExporter = operationLogsExporter; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -68,7 +72,7 @@ public AsyncExecutionState execute(ProcessContext context) { return AsyncExecutionState.FINISHED; } if (context.getVariable(Variables.UPLOAD_START_TIME) == null) { - context.setVariable(Variables.UPLOAD_START_TIME, LocalDateTime.now()); + context.setVariable(Variables.UPLOAD_START_TIME, Instant.now()); } ApplicationToUploadContext applicationToUploadContext = buildApplicationToUploadContext(context, applicationToProcess); CloudControllerClient client = context.getControllerClient(); @@ -76,6 +80,7 @@ public AsyncExecutionState execute(ProcessContext context) { try { runningUpload = appUploaderThreadPool.submit(() -> doUpload(context, applicationToProcess, applicationToUploadContext)); } catch (RejectedExecutionException rejectedExecutionException) { + uploadDurationTracker.recordAppBinaryUploadRejection(); LOGGER.warn(rejectedExecutionException.getMessage(), rejectedExecutionException); context.getStepLogger() .warn(Messages.UPLOAD_OF_APPLICATION_0_WAS_NOT_ACCEPTED_BY_INSTANCE_1, applicationToProcess.getName(), @@ -86,13 +91,12 @@ public AsyncExecutionState execute(ProcessContext context) { try { cloudPackage = runningUpload.get(); } catch (InterruptedException | ExecutionException e) { + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new SLException(e, e.getMessage()); } - LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); - long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) - .toMillis(); + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), false); context.getStepLogger() - .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, timeElapsedForUpload); + .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(context)); return processCloudPackage(context, client, cloudPackage); } @@ -149,10 +153,10 @@ private CloudPackage asyncUploadFiles(CloudControllerClient client, ApplicationT } private Path extractApplicationFromArchive(ApplicationToUploadContext applicationToUploadContext) { - LocalDateTime startTime = LocalDateTime.now(); + Instant startTime = Instant.now(); Path extractedAppPath = extractFromMtar( createApplicationArchiveContext(applicationToUploadContext, applicationConfiguration.getMaxResourceFileSize())); - long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) + long timeElapsedForUpload = Duration.between(startTime, Instant.now()) .toMillis(); applicationToUploadContext.getStepLogger() .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_APP_BINARY_DOWNLOAD_0_IN_MILLIS, @@ -210,6 +214,12 @@ private void attemptToUpdateApplicationDigest(CloudControllerClient client, Clou newApplicationDigest); } + private long getElapsedTimeInMillis(ProcessContext context) { + Instant startTime = context.getVariable(Variables.UPLOAD_START_TIME); + return Duration.between(startTime, Instant.now()) + .toMillis(); + } + @Override public String getPollingErrorMessage(ProcessContext context) { CloudApplicationExtended appToProcess = context.getVariable(Variables.APP_TO_PROCESS); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java index 3b0cbaa25a..3874382dbb 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java @@ -21,6 +21,7 @@ import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationFileDigestDetector; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.security.serialization.DynamicSecureSerialization; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.security.util.SecureLoggingUtil; @@ -53,6 +54,8 @@ public class UploadAppStep extends TimeoutAsyncFlowableStep { protected CloudPackagesGetter cloudPackagesGetter; @Inject private ExecutorService appUploaderThreadPool; + @Inject + private UploadDurationTracker uploadDurationTracker; @Override public StepPhase executeAsyncStep(ProcessContext context) throws FileStorageException { @@ -186,7 +189,7 @@ private void removeApplicationDigestIfSet(ProcessContext context, Map getAsyncStepExecutions(ProcessContext context) { return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool, - operationLogsExporter), + operationLogsExporter, uploadDurationTracker), new PollUploadAppStatusExecution()); } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java index e498355147..efe9f0f026 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java @@ -683,7 +683,7 @@ public interface Variables { .name("waitBindServiceTimeout") .defaultValue(Duration.ofHours(1)) .build(); - Variable UPLOAD_START_TIME = ImmutableSimpleVariable. builder() + Variable UPLOAD_START_TIME = ImmutableSimpleVariable. builder() .name("uploadStartTime") .build(); diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java index 4e832ac010..d831d3b2f0 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java @@ -29,6 +29,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableLoggingConfiguration; import org.cloudfoundry.multiapps.controller.persistence.model.LogLevel; import org.cloudfoundry.multiapps.controller.persistence.model.LoggingConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveIterator; @@ -92,6 +93,7 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest .build(); private final MtaArchiveElements mtaArchiveElements = new MtaArchiveElements(); private final ExecutorService appUploaderThreadPool = mock(ExecutorService.class); + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @TempDir Path tempDir; @@ -212,8 +214,9 @@ void testSendApplicationLogsToCloudLoggingService_sendsLogsForEachProcessLog() { context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, loggingConfiguration); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(List.of("log-1", "log-2")); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(List.of("log-1", "log-2")); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -230,8 +233,9 @@ void testSendApplicationLogsToCloudLoggingService_withNullLoggingConfiguration_s context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, null); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(List.of("log-1")); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(List.of("log-1")); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -248,8 +252,9 @@ void testSendApplicationLogsToCloudLoggingService_withNoLogs_doesNotSendProcessL context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, loggingConfiguration); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(Collections.emptyList()); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(Collections.emptyList()); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -266,8 +271,9 @@ void testSendApplicationLogsToCloudLoggingService_notCalledWhenStatusIsNotReady( context.setVariable(Variables.EXTERNAL_LOGGING_SERVICE_CONFIGURATION, loggingConfiguration); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); - when(step.getProcessLogsPersister().getApplicationProcessLogsMessages(TEST_CORRELATION_ID, - TEST_TASK_ID)).thenReturn(List.of("process-log-content")); + when(step.getProcessLogsPersister() + .getApplicationProcessLogsMessages(TEST_CORRELATION_ID, + TEST_TASK_ID)).thenReturn(List.of("process-log-content")); expectedStatus = AsyncExecutionState.FINISHED; testExecuteOperations(); @@ -314,7 +320,7 @@ protected List getAsyncStepExecutions(ProcessContext context) { getProcessLogsPersister(), configuration, appUploaderThreadPool, - operationLogsExporter) { + operationLogsExporter, uploadDurationTracker) { }); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java index c892373baa..85cdcb3c31 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java @@ -1,9 +1,11 @@ package org.cloudfoundry.multiapps.controller.web.configuration; +import jakarta.inject.Inject; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.NoRetryErrorClassifier; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.RetryableErrorClassifier; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.configuration.factory.ObjectStoreSelectorFactory; import org.springframework.context.annotation.Bean; @@ -16,10 +18,14 @@ public class FileStorageConfiguration { private static final String OBJECT_STORE_SERVICE_NAME = "deploy-service-os"; + @Inject + private UploadDurationTracker uploadDurationTracker; + @Bean public ObjectStoreSelectorFactory objectStoreSelector(EnvironmentServicesFinder vcapServiceFinder, ApplicationConfiguration applicationConfiguration) { - return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration); + return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration, + uploadDurationTracker); } @Bean diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java index cb52bb629a..a8aca8efc2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java @@ -3,6 +3,7 @@ import java.util.Map; import org.cloudfoundry.multiapps.controller.web.monitoring.Metrics; +import org.cloudfoundry.multiapps.controller.web.monitoring.UploadDurationMetrics; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jmx.export.MBeanExporter; @@ -12,10 +13,12 @@ public class JmxConfiguration { private static final String METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=MetricsMBean"; + private static final String UPLOAD_METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=UploadMetricsMBean"; + @Bean - public MBeanExporter jmxExporter(Metrics metrics) { + public MBeanExporter jmxExporter(Metrics metrics, UploadDurationMetrics uploadDurationMetrics) { MBeanExporter mBeanExporter = new MBeanExporter(); - mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics)); + mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics, UPLOAD_METRICS_BEAN, uploadDurationMetrics)); return mBeanExporter; } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java index 117ed389ad..ceeb850bf2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java @@ -7,11 +7,13 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; + import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; @@ -39,12 +41,15 @@ public class ObjectStoreSelectorFactory { private final EnvironmentServicesFinder environmentServicesFinder; private final ApplicationConfiguration applicationConfiguration; private final SelectedObjectStore selectedObjectStore; + private final UploadDurationTracker uploadDurationTracker; public ObjectStoreSelectorFactory(String serviceName, EnvironmentServicesFinder environmentServicesFinder, - ApplicationConfiguration applicationConfiguration) { + ApplicationConfiguration applicationConfiguration, + UploadDurationTracker uploadDurationTracker) { this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; this.applicationConfiguration = applicationConfiguration; + this.uploadDurationTracker = uploadDurationTracker; this.selectedObjectStore = doSelect(); } @@ -155,15 +160,15 @@ private SelectedObjectStore selectFor(ObjectStoreServiceInfo objectStoreServiceI } protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -203,7 +208,7 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { return new JCloudsObjectStoreFileStorage(context.getBlobStore(), (String) objectStoreServiceInfo.getCredentials() - .get(Constants.BUCKET)); + .get(Constants.BUCKET), uploadDurationTracker); } private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java new file mode 100644 index 0000000000..3207495a23 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java @@ -0,0 +1,28 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +@Named +public class AppUploaderThreadPoolInformation { + + private final ThreadPoolExecutor appUploaderPool; + + @Inject + public AppUploaderThreadPoolInformation( + @Named("appUploaderThreadPool") ExecutorService appUploaderThreadPool) { + this.appUploaderPool = (ThreadPoolExecutor) appUploaderThreadPool; + } + + public int getActiveThreads() { + return appUploaderPool.getActiveCount(); + } + + public int getMaxThreads() { + return appUploaderPool.getMaximumPoolSize(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java new file mode 100644 index 0000000000..36e111505b --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java @@ -0,0 +1,103 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; + +@Named +public class UploadDurationMetrics implements UploadMetricsMBean { + + private final UploadDurationTracker tracker; + + private final AppUploaderThreadPoolInformation appUploaderInfo; + + @Inject + public UploadDurationMetrics(UploadDurationTracker tracker, + AppUploaderThreadPoolInformation appUploaderInfo) { + this.tracker = tracker; + this.appUploaderInfo = appUploaderInfo; + } + + @Override + public long getAppBinaryUploadTotalCount() { + return tracker.getAppBinaryStatistics() + .totalCount(); + } + + @Override + public long getAppBinaryUploadTimeoutCount() { + return tracker.getAppBinaryStatistics() + .timeoutCount(); + } + + @Override + public long getAppBinaryUploadMaxDurationMs() { + return tracker.getAppBinaryStatistics() + .maxDurationMs(); + } + + @Override + public long getAppBinaryUploadSumDurationMs() { + return tracker.getAppBinaryStatistics() + .sumDurationMs(); + } + + @Override + public int getAppUploaderActiveThreads() { + return appUploaderInfo.getActiveThreads(); + } + + @Override + public int getAppUploaderMaxThreads() { + return appUploaderInfo.getMaxThreads(); + } + + @Override + public long getAppUploaderRejectionCount() { + return tracker.getAppBinaryStatistics() + .rejectionCount(); + } + + @Override + public long getAppUploaderRejectionCountInWindow() { + return tracker.getAppBinaryStatistics() + .rejectionsInWindow(); + } + + @Override + public long getObjectStoreUploadTotalCount() { + return tracker.getObjectStoreStatistics() + .totalCount(); + } + + @Override + public long getObjectStoreUploadTimeoutCount() { + return tracker.getObjectStoreStatistics() + .timeoutCount(); + } + + @Override + public long getObjectStoreUploadMaxDurationMs() { + return tracker.getObjectStoreStatistics() + .maxDurationMs(); + } + + @Override + public long getObjectStoreUploadSumDurationMs() { + return tracker.getObjectStoreStatistics() + .sumDurationMs(); + } + + @Override + public long getAppBinaryUploadTimeoutsInWindow() { + return tracker.getAppBinaryStatistics() + .timeoutsInWindow(); + } + + @Override + public long getObjectStoreUploadTimeoutsInWindow() { + return tracker.getObjectStoreStatistics() + .timeoutsInWindow(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java new file mode 100644 index 0000000000..396c352db1 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java @@ -0,0 +1,33 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +public interface UploadMetricsMBean { + + long getAppBinaryUploadTotalCount(); + + long getAppBinaryUploadTimeoutCount(); + + long getAppBinaryUploadMaxDurationMs(); + + long getAppBinaryUploadSumDurationMs(); + + int getAppUploaderActiveThreads(); + + int getAppUploaderMaxThreads(); + + long getAppUploaderRejectionCount(); + + long getAppUploaderRejectionCountInWindow(); + + long getObjectStoreUploadTotalCount(); + + long getObjectStoreUploadTimeoutCount(); + + long getObjectStoreUploadMaxDurationMs(); + + long getObjectStoreUploadSumDurationMs(); + + long getAppBinaryUploadTimeoutsInWindow(); + + long getObjectStoreUploadTimeoutsInWindow(); + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java index b28570c678..911f956f89 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java @@ -4,6 +4,7 @@ import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -64,6 +65,8 @@ class ObjectStoreSelectorFactoryTest { private AwsS3ObjectStoreFileStorage awsS3ObjectStoreFileStorage; @Mock private AzureObjectStoreFileStorage azureObjectStoreFileStorage; + @Mock + private UploadDurationTracker uploadDurationTracker; private ObjectStoreServiceInfo capturedGcpServiceInfo; private ObjectStoreServiceInfo capturedAzureServiceInfo; @@ -103,7 +106,7 @@ void testObjectStoreCreationWhenEnvIsValidForAws() { assertInstanceOf(AwsS3ObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AwsTransientErrorClassifier.class, selector.classifier()); verify(awsS3ObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -116,7 +119,7 @@ void testObjectStoreCreationWhenEnvIsValidForAzure() { assertInstanceOf(AzureObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AzureTransientErrorClassifier.class, selector.classifier()); verify(azureObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -129,12 +132,12 @@ void testObjectStoreCreationWhenEnvIsValidForGcp() { assertInstanceOf(GcpObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(GcpTransientErrorClassifier.class, selector.classifier()); verify(gcpObjectStoreFileStorage) - .testConnection(); + .testConnection(); } static Stream testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid() { return Stream.of( - // @formatter:off + // @formatter:off // (0) Unknown provider name: Arguments.of("WRONG_PROVIDER"), // (1) Null env value: @@ -169,7 +172,7 @@ void testObjectStoreCreationFallsBackToNextProviderWhenFirstFails() { assertInstanceOf(JCloudsObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(JCloudsTransientErrorClassifier.class, selector.classifier()); verify(jCloudsObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -289,7 +292,7 @@ private class ObjectStoreSelectorFactoryMock extends ObjectStoreSelectorFactory ObjectStoreSelectorFactoryMock(String serviceName, EnvironmentServicesFinder environmentServicesFinder, ApplicationConfiguration applicationConfiguration) { - super(serviceName, environmentServicesFinder, applicationConfiguration); + super(serviceName, environmentServicesFinder, applicationConfiguration, uploadDurationTracker); } @Override diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java new file mode 100644 index 0000000000..c9f55efada --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformationTest.java @@ -0,0 +1,40 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class AppUploaderThreadPoolInformationTest { + + private ThreadPoolExecutor threadPoolExecutor; + private AppUploaderThreadPoolInformation appUploaderThreadPoolInformation; + + @BeforeEach + void setUp() { + threadPoolExecutor = new ThreadPoolExecutor(2, 5, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + appUploaderThreadPoolInformation = new AppUploaderThreadPoolInformation(threadPoolExecutor); + } + + @AfterEach + void cleanUp() { + threadPoolExecutor.shutdownNow(); + } + + @Test + void testGetActiveThreadsReturnsActiveCount() { + assertEquals(threadPoolExecutor.getActiveCount(), appUploaderThreadPoolInformation.getActiveThreads()); + } + + @Test + void testGetMaxThreadsReturnsMaximumPoolSize() { + assertEquals(5, appUploaderThreadPoolInformation.getMaxThreads()); + assertEquals(threadPoolExecutor.getMaximumPoolSize(), appUploaderThreadPoolInformation.getMaxThreads()); + } + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java new file mode 100644 index 0000000000..6826959f7a --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetricsTest.java @@ -0,0 +1,167 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker.UploadPathStatistics; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class UploadDurationMetricsTest { + + @Mock + private UploadDurationTracker uploadDurationTracker; + + @Mock + private UploadPathStatistics appBinaryStatistics; + + @Mock + private UploadPathStatistics objectStoreStatistics; + + @Mock + private AppUploaderThreadPoolInformation appUploaderInfo; + + private UploadDurationMetrics uploadDurationMetrics; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + when(uploadDurationTracker.getAppBinaryStatistics()) + .thenReturn(appBinaryStatistics); + when(uploadDurationTracker.getObjectStoreStatistics()) + .thenReturn(objectStoreStatistics); + uploadDurationMetrics = new UploadDurationMetrics(uploadDurationTracker, appUploaderInfo); + } + + @Test + void testGetAppBinaryUploadTotalCount() { + when(appBinaryStatistics.totalCount()) + .thenReturn(11L); + assertEquals(11L, uploadDurationMetrics.getAppBinaryUploadTotalCount()); + verify(appBinaryStatistics) + .totalCount(); + } + + @Test + void testGetAppBinaryUploadTimeoutCount() { + when(appBinaryStatistics.timeoutCount()) + .thenReturn(12L); + assertEquals(12L, uploadDurationMetrics.getAppBinaryUploadTimeoutCount()); + verify(appBinaryStatistics) + .timeoutCount(); + } + + @Test + void testGetAppBinaryUploadMaxDurationMs() { + when(appBinaryStatistics.maxDurationMs()) + .thenReturn(13L); + assertEquals(13L, uploadDurationMetrics.getAppBinaryUploadMaxDurationMs()); + verify(appBinaryStatistics) + .maxDurationMs(); + } + + @Test + void testGetAppBinaryUploadSumDurationMs() { + when(appBinaryStatistics.sumDurationMs()) + .thenReturn(14L); + assertEquals(14L, uploadDurationMetrics.getAppBinaryUploadSumDurationMs()); + verify(appBinaryStatistics) + .sumDurationMs(); + } + + @Test + void testGetAppUploaderActiveThreads() { + when(appUploaderInfo.getActiveThreads()) + .thenReturn(3); + assertEquals(3, uploadDurationMetrics.getAppUploaderActiveThreads()); + verify(appUploaderInfo) + .getActiveThreads(); + } + + @Test + void testGetAppUploaderMaxThreads() { + when(appUploaderInfo.getMaxThreads()) + .thenReturn(7); + assertEquals(7, uploadDurationMetrics.getAppUploaderMaxThreads()); + verify(appUploaderInfo) + .getMaxThreads(); + } + + @Test + void testGetAppUploaderRejectionCount() { + when(appBinaryStatistics.rejectionCount()) + .thenReturn(15L); + assertEquals(15L, uploadDurationMetrics.getAppUploaderRejectionCount()); + verify(appBinaryStatistics) + .rejectionCount(); + } + + @Test + void testGetAppUploaderRejectionCountInWindow() { + when(appBinaryStatistics.rejectionsInWindow()) + .thenReturn(16L); + assertEquals(16L, uploadDurationMetrics.getAppUploaderRejectionCountInWindow()); + verify(appBinaryStatistics) + .rejectionsInWindow(); + } + + @Test + void testGetObjectStoreUploadTotalCount() { + when(objectStoreStatistics.totalCount()) + .thenReturn(21L); + assertEquals(21L, uploadDurationMetrics.getObjectStoreUploadTotalCount()); + verify(objectStoreStatistics) + .totalCount(); + } + + @Test + void testGetObjectStoreUploadTimeoutCount() { + when(objectStoreStatistics.timeoutCount()) + .thenReturn(22L); + assertEquals(22L, uploadDurationMetrics.getObjectStoreUploadTimeoutCount()); + verify(objectStoreStatistics) + .timeoutCount(); + } + + @Test + void testGetObjectStoreUploadMaxDurationMs() { + when(objectStoreStatistics.maxDurationMs()) + .thenReturn(23L); + assertEquals(23L, uploadDurationMetrics.getObjectStoreUploadMaxDurationMs()); + verify(objectStoreStatistics) + .maxDurationMs(); + } + + @Test + void testGetObjectStoreUploadSumDurationMs() { + when(objectStoreStatistics.sumDurationMs()) + .thenReturn(24L); + assertEquals(24L, uploadDurationMetrics.getObjectStoreUploadSumDurationMs()); + verify(objectStoreStatistics) + .sumDurationMs(); + } + + @Test + void testGetAppBinaryUploadTimeoutsInWindow() { + when(appBinaryStatistics.timeoutsInWindow()) + .thenReturn(17L); + assertEquals(17L, uploadDurationMetrics.getAppBinaryUploadTimeoutsInWindow()); + verify(appBinaryStatistics) + .timeoutsInWindow(); + } + + @Test + void testGetObjectStoreUploadTimeoutsInWindow() { + when(objectStoreStatistics.timeoutsInWindow()) + .thenReturn(25L); + assertEquals(25L, uploadDurationMetrics.getObjectStoreUploadTimeoutsInWindow()); + verify(objectStoreStatistics) + .timeoutsInWindow(); + } + +}