diff --git a/.kokoro/bigtable-cleanup.sh b/.kokoro/bigtable-cleanup.sh new file mode 100755 index 000000000000..f333a297c728 --- /dev/null +++ b/.kokoro/bigtable-cleanup.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +## Get the directory of the build script +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +## cd to the parent directory, i.e. the root of the git repo +cd ${scriptDir}/.. + +# include common functions +source ${scriptDir}/common.sh + +# Use GCP Maven Mirror +mkdir -p ${HOME}/.m2 +cp settings.xml ${HOME}/.m2 + +setup_application_credentials + +echo "Compiling and building all modules for java-bigtable" +install_modules "java-bigtable" + +pushd java-bigtable +RETURN_CODE=0 +for instance in google-cloud-bigtable google-cloud-bigtable-session; do + echo "Cleaning up stale resources in instance: ${instance}" + mvn -B -ntp -pl google-cloud-bigtable \ + exec:java@cleanup-stale-resources \ + -Dbigtable.project="${GOOGLE_CLOUD_PROJECT:-gcloud-devel}" \ + -Dbigtable.instance="${instance}" \ + -Dbigtable.table=integration-tests || RETURN_CODE=$? +done +popd + +echo "exiting with ${RETURN_CODE}" +exit ${RETURN_CODE} diff --git a/.kokoro/nightly/bigtable-cleanup.cfg b/.kokoro/nightly/bigtable-cleanup.cfg new file mode 100644 index 000000000000..3811fdbfac82 --- /dev/null +++ b/.kokoro/nightly/bigtable-cleanup.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-java/.kokoro/bigtable-cleanup.sh" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} diff --git a/java-bigtable/google-cloud-bigtable/pom.xml b/java-bigtable/google-cloud-bigtable/pom.xml index b22256648258..086cd11656da 100644 --- a/java-bigtable/google-cloud-bigtable/pom.xml +++ b/java-bigtable/google-cloud-bigtable/pom.xml @@ -485,7 +485,6 @@ enable-verbose-grpc-logs - true src/test/resources/logging-verbose.properties @@ -527,8 +526,6 @@ emulator - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/emulator-it com.google.cloud.bigtable.**.it.*IT @@ -564,8 +561,6 @@ ${bigtable.cfe-data-endpoint} ${bigtable.cfe-admin-endpoint} ${bigtable.cfe-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/prod-it com.google.cloud.bigtable.**.it.*IT @@ -608,8 +603,6 @@ ${bigtable.directpath-data-endpoint} ${bigtable.directpath-admin-endpoint} ${bigtable.directpath-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/directpath-it REQUIRE_DIRECT_PATH @@ -650,8 +643,6 @@ ${bigtable.cfe-data-endpoint} ${bigtable.cfe-admin-endpoint} ${bigtable.cfe-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/cfe-it REQUIRE_CFE @@ -689,8 +680,6 @@ ${bigtable.directpath-data-endpoint} ${bigtable.directpath-admin-endpoint} ${bigtable.directpath-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/directpath-ipv4only-it REQUIRE_DIRECT_PATH_IPV4 @@ -818,6 +807,51 @@ + + org.codehaus.mojo + exec-maven-plugin + + + + cleanup-stale-resources + + com.google.cloud.bigtable.test_helpers.env.StaleResourceCleaner + test + + + java.util.logging.config.file + ${java-log-config} + + + bigtable.data-endpoint + ${bigtable.cfe-data-endpoint} + + + bigtable.admin-endpoint + ${bigtable.cfe-admin-endpoint} + + + bigtable.data-jwt-audience + ${bigtable.cfe-jwt-audience} + + + + + + + org.codehaus.mojo flatten-maven-plugin diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java new file mode 100644 index 000000000000..ab067975738b --- /dev/null +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java @@ -0,0 +1,452 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.test_helpers.env; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.AppProfile; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.LogicalView; +import com.google.cloud.bigtable.admin.v2.models.MaterializedView; +import com.google.cloud.bigtable.admin.v2.models.Table; +import com.google.cloud.bigtable.admin.v2.models.UpdateAuthorizedViewRequest; +import com.google.cloud.bigtable.admin.v2.models.UpdateLogicalViewRequest; +import com.google.cloud.bigtable.admin.v2.models.UpdateMaterializedViewRequest; +import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Standalone utility to clean up stale test resources older than 24 hours in the configured {@link + * CloudEnv} shared Bigtable instance as well as dynamically created test instances in the project. + */ +public class StaleResourceCleaner { + + private static final Logger LOGGER = Logger.getLogger(StaleResourceCleaner.class.getName()); + + private final AbstractTestEnv testEnv; + private final List errors = new ArrayList<>(); + + StaleResourceCleaner(AbstractTestEnv testEnv) { + this.testEnv = testEnv; + } + + public static void main(String[] args) throws Exception { + CloudEnv cloudEnv = CloudEnv.fromSystemProperties(); + cloudEnv.start(); + try { + new StaleResourceCleaner(cloudEnv).cleanUp(); + } finally { + cloudEnv.stop(); + } + } + + static boolean isStale(String resourceId, String stalePrefix) { + return resourceId.startsWith(PrefixGenerator.PREFIX) && resourceId.compareTo(stalePrefix) < 0; + } + + public void cleanUp() { + String stalePrefix = PrefixGenerator.newTimePrefix(Instant.now().minus(1, ChronoUnit.DAYS)); + cleanUp(stalePrefix); + } + + void cleanUp(String stalePrefix) { + errors.clear(); + String instanceId = testEnv.getInstanceId(); + LOGGER.info( + "Starting stale resource cleanup with stalePrefix=" + + stalePrefix + + " for shared instance=" + + instanceId); + cleanUpSharedInstance(instanceId, stalePrefix); + cleanUpStaleInstances(stalePrefix); + + if (!errors.isEmpty()) { + RuntimeException exception = + new RuntimeException( + "Stale resource cleanup encountered " + errors.size() + " error(s)", errors.get(0)); + for (int i = 1; i < errors.size(); i++) { + exception.addSuppressed(errors.get(i)); + } + throw exception; + } + } + + void cleanUpSharedInstance(String instanceId, String stalePrefix) { + LOGGER.info("Cleaning up shared instance: " + instanceId); + BigtableTableAdminClient tableAdmin = testEnv.getTableAdminClient(); + cleanUpStaleMaterializedViews(instanceId, stalePrefix); + cleanUpStaleLogicalViews(instanceId, stalePrefix); + cleanUpTablesAndAuthorizedViews(tableAdmin, stalePrefix); + cleanUpStaleAppProfiles(instanceId, stalePrefix); + cleanUpClustersAndBackups(instanceId, tableAdmin, stalePrefix); + } + + private void cleanUpStaleMaterializedViews(String instanceId, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + List views; + try { + views = instanceAdmin.listMaterializedViews(instanceId); + } catch (Exception e) { + recordError("Failed to list materialized views in instance: " + instanceId, e); + return; + } + for (MaterializedView view : views) { + if (!isStale(view.getId(), stalePrefix)) { + continue; + } + LOGGER.info( + "Deleting stale materialized view: " + view.getId() + " in instance " + instanceId); + unprotectAndDeleteMaterializedView(instanceId, view); + } + } + + private void unprotectAndDeleteMaterializedView(String instanceId, MaterializedView view) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + try { + if (view.isDeletionProtected()) { + instanceAdmin.updateMaterializedView( + UpdateMaterializedViewRequest.of(instanceId, view.getId()) + .setDeletionProtection(false)); + } + instanceAdmin.deleteMaterializedView(instanceId, view.getId()); + } catch (Exception e) { + recordError( + "Failed to delete stale materialized view: " + + view.getId() + + " in instance " + + instanceId, + e); + } + } + + private void cleanUpStaleLogicalViews(String instanceId, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + List views; + try { + views = instanceAdmin.listLogicalViews(instanceId); + } catch (Exception e) { + recordError("Failed to list logical views in instance: " + instanceId, e); + return; + } + for (LogicalView view : views) { + if (!isStale(view.getId(), stalePrefix)) { + continue; + } + LOGGER.info("Deleting stale logical view: " + view.getId() + " in instance " + instanceId); + unprotectAndDeleteLogicalView(instanceId, view); + } + } + + private void unprotectAndDeleteLogicalView(String instanceId, LogicalView view) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + try { + if (view.isDeletionProtected()) { + instanceAdmin.updateLogicalView( + UpdateLogicalViewRequest.of(instanceId, view.getId()).setDeletionProtection(false)); + } + instanceAdmin.deleteLogicalView(instanceId, view.getId()); + } catch (Exception e) { + recordError( + "Failed to delete stale logical view: " + view.getId() + " in instance " + instanceId, e); + } + } + + private void cleanUpTablesAndAuthorizedViews( + BigtableTableAdminClient tableAdmin, String stalePrefix) { + LOGGER.info("Start cleaning up stale tables with stalePrefix=" + stalePrefix); + List tableIds; + try { + tableIds = tableAdmin.listTables(); + } catch (Exception e) { + recordError("Failed to list tables", e); + return; + } + for (String tableId : tableIds) { + if (!tableId.startsWith(PrefixGenerator.PREFIX)) { + cleanUpStaleAuthorizedViewsOnPersistentTable(tableAdmin, tableId, stalePrefix); + continue; + } + if (isStale(tableId, stalePrefix)) { + LOGGER.info("Preparing stale table for delete: " + tableId); + try { + prepTableForDelete(tableAdmin, tableId); + LOGGER.info("Deleting stale table: " + tableId); + tableAdmin.deleteTable(tableId); + } catch (Exception e) { + recordError("Deleting stale table failed: " + tableId, e); + } + } + } + } + + private void prepTableForDelete(BigtableTableAdminClient tableAdmin, String tableId) { + Table table = tableAdmin.getTable(tableId); + if (table.isDeletionProtected() || table.getChangeStreamRetention() != null) { + tableAdmin.updateTable( + UpdateTableRequest.of(tableId) + .setDeletionProtection(false) + .disableChangeStreamRetention()); + } + + for (String viewId : tableAdmin.listAuthorizedViews(tableId)) { + try { + tableAdmin.updateAuthorizedView( + UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); + } catch (Exception e) { + recordError( + "Failed to unprotect authorized view: " + viewId + " on stale table: " + tableId, e); + } + } + } + + private void cleanUpStaleAuthorizedViewsOnPersistentTable( + BigtableTableAdminClient tableAdmin, String tableId, String stalePrefix) { + List viewIds; + try { + viewIds = tableAdmin.listAuthorizedViews(tableId); + } catch (Exception e) { + recordError("Failed to list authorized views on persistent table: " + tableId, e); + return; + } + for (String viewId : viewIds) { + if (!isStale(viewId, stalePrefix)) { + continue; + } + LOGGER.info("Deleting stale authorized view: " + viewId + " on persistent table: " + tableId); + try { + tableAdmin.updateAuthorizedView( + UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); + tableAdmin.deleteAuthorizedView(tableId, viewId); + } catch (Exception e) { + recordError( + "Failed to delete stale authorized view: " + + viewId + + " on persistent table: " + + tableId, + e); + } + } + } + + private void cleanUpStaleAppProfiles(String instanceId, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + List appProfiles; + try { + appProfiles = instanceAdmin.listAppProfiles(instanceId); + } catch (Exception e) { + recordError("Failed to list app profiles in instance: " + instanceId, e); + return; + } + for (AppProfile appProfile : appProfiles) { + if (!isStale(appProfile.getId(), stalePrefix)) { + continue; + } + try { + instanceAdmin.deleteAppProfile(instanceId, appProfile.getId(), true); + } catch (Exception e) { + recordError( + "Failed to delete stale app profile: " + + appProfile.getId() + + " in instance " + + instanceId, + e); + } + } + } + + private void cleanUpClustersAndBackups( + String instanceId, BigtableTableAdminClient tableAdmin, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + List clusters; + try { + clusters = instanceAdmin.listClusters(instanceId); + } catch (Exception e) { + recordError("Failed to list clusters in instance: " + instanceId, e); + return; + } + for (Cluster cluster : clusters) { + if (!cluster.getId().startsWith(PrefixGenerator.PREFIX)) { + try { + deleteStaleBackups(tableAdmin, cluster.getId(), stalePrefix); + } catch (Exception e) { + recordError("Failed to delete stale backups in cluster: " + cluster.getId(), e); + } + continue; + } + if (!isStale(cluster.getId(), stalePrefix)) { + continue; + } + + try { + deleteAllBackups(tableAdmin, cluster.getId()); + } catch (Exception e) { + recordError("Failed to delete backups in stale cluster: " + cluster.getId(), e); + } + try { + instanceAdmin.deleteCluster(instanceId, cluster.getId()); + } catch (Exception e) { + recordError( + "Failed to delete stale cluster: " + cluster.getId() + " in instance " + instanceId, e); + } + } + } + + private void cleanUpStaleInstances(String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + List instances; + try { + instances = instanceAdmin.listInstances(); + } catch (Exception e) { + recordError("Failed to list instances", e); + return; + } + for (Instance instance : instances) { + if (!isStale(instance.getId(), stalePrefix)) { + continue; + } + try { + prepInstanceForDelete(instance.getId()); + deleteInstance(instance.getId()); + } catch (Exception e) { + recordError("Failed to delete stale instance: " + instance.getId(), e); + } + } + } + + private void prepInstanceForDelete(String instanceId) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + try { + for (MaterializedView materializedView : instanceAdmin.listMaterializedViews(instanceId)) { + unprotectAndDeleteMaterializedView(instanceId, materializedView); + } + } catch (NotFoundException e) { + throw e; + } catch (Exception e) { + recordError("Failed to list materialized views in stale instance: " + instanceId, e); + } + try { + for (LogicalView logicalView : instanceAdmin.listLogicalViews(instanceId)) { + unprotectAndDeleteLogicalView(instanceId, logicalView); + } + } catch (NotFoundException e) { + throw e; + } catch (Exception e) { + recordError("Failed to list logical views in stale instance: " + instanceId, e); + } + } + + private void deleteInstance(String instanceId) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + try (BigtableTableAdminClient tableAdmin = testEnv.getTableAdminClientForInstance(instanceId)) { + List clusters = instanceAdmin.listClusters(instanceId); + + boolean isFirstCluster = true; + + for (Cluster cluster : clusters) { + try { + deleteAllBackups(tableAdmin, cluster.getId()); + } catch (Exception e) { + recordError( + "Failed to delete backups in cluster " + + cluster.getId() + + " of stale instance " + + instanceId, + e); + } + // Skip the first cluster so that it can be deleted by deleteInstance (instances can't exist + // without clusters) + if (!isFirstCluster) { + try { + instanceAdmin.deleteCluster(instanceId, cluster.getId()); + } catch (Exception e) { + recordError( + "Failed to delete cluster " + cluster.getId() + " of stale instance " + instanceId, + e); + } + } + isFirstCluster = false; + } + } catch (Exception e) { + recordError("Failed to clean up clusters/backups in stale instance: " + instanceId, e); + } + + try { + instanceAdmin.deleteInstance(instanceId); + } catch (Exception e) { + recordError("Failed to delete stale instance: " + instanceId, e); + } + } + + private void deleteAllBackups(BigtableTableAdminClient tableAdmin, String clusterId) { + List> futures = new ArrayList<>(); + + for (String backupId : tableAdmin.listBackups(clusterId)) { + ApiFuture f = tableAdmin.deleteBackupAsync(clusterId, backupId); + futures.add(f); + } + + awaitBackupDeletions(futures, clusterId); + } + + private void deleteStaleBackups( + BigtableTableAdminClient tableAdmin, String clusterId, String stalePrefix) { + List> futures = new ArrayList<>(); + + for (String backupId : tableAdmin.listBackups(clusterId)) { + if (isStale(backupId, stalePrefix)) { + ApiFuture f = tableAdmin.deleteBackupAsync(clusterId, backupId); + futures.add(f); + } + } + + awaitBackupDeletions(futures, clusterId); + } + + private void awaitBackupDeletions(List> futures, String clusterId) { + for (ApiFuture future : futures) { + try { + future.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof NotFoundException) { + continue; + } + recordError("Failed to delete backup in cluster: " + clusterId, e); + } catch (Exception e) { + recordError("Failed to delete backup in cluster: " + clusterId, e); + } + } + } + + private void recordError(String message, Exception e) { + if (e instanceof NotFoundException) { + return; + } + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + LOGGER.log(Level.WARNING, message, e); + errors.add(e); + } +} diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java new file mode 100644 index 000000000000..86a54802fc93 --- /dev/null +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java @@ -0,0 +1,1094 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.test_helpers.env; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import com.google.api.core.ApiFutures; +import com.google.api.gax.core.BackgroundResource; +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.longrunning.OperationFuture; +import com.google.api.gax.longrunning.OperationFutures; +import com.google.api.gax.longrunning.OperationSnapshot; +import com.google.api.gax.rpc.InternalException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.OperationCallable; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.api.gax.rpc.testing.FakeOperationSnapshot; +import com.google.bigtable.admin.v2.AppProfile; +import com.google.bigtable.admin.v2.AuthorizedView; +import com.google.bigtable.admin.v2.Backup; +import com.google.bigtable.admin.v2.ChangeStreamConfig; +import com.google.bigtable.admin.v2.Cluster; +import com.google.bigtable.admin.v2.DeleteAppProfileRequest; +import com.google.bigtable.admin.v2.DeleteAuthorizedViewRequest; +import com.google.bigtable.admin.v2.DeleteBackupRequest; +import com.google.bigtable.admin.v2.DeleteClusterRequest; +import com.google.bigtable.admin.v2.DeleteInstanceRequest; +import com.google.bigtable.admin.v2.DeleteLogicalViewRequest; +import com.google.bigtable.admin.v2.DeleteMaterializedViewRequest; +import com.google.bigtable.admin.v2.DeleteTableRequest; +import com.google.bigtable.admin.v2.GetTableRequest; +import com.google.bigtable.admin.v2.Instance; +import com.google.bigtable.admin.v2.ListAppProfilesRequest; +import com.google.bigtable.admin.v2.ListAuthorizedViewsRequest; +import com.google.bigtable.admin.v2.ListBackupsRequest; +import com.google.bigtable.admin.v2.ListClustersRequest; +import com.google.bigtable.admin.v2.ListClustersResponse; +import com.google.bigtable.admin.v2.ListInstancesRequest; +import com.google.bigtable.admin.v2.ListInstancesResponse; +import com.google.bigtable.admin.v2.ListLogicalViewsRequest; +import com.google.bigtable.admin.v2.ListMaterializedViewsRequest; +import com.google.bigtable.admin.v2.ListTablesRequest; +import com.google.bigtable.admin.v2.LogicalView; +import com.google.bigtable.admin.v2.MaterializedView; +import com.google.bigtable.admin.v2.Table; +import com.google.bigtable.admin.v2.Table.View; +import com.google.bigtable.admin.v2.UpdateAuthorizedViewMetadata; +import com.google.bigtable.admin.v2.UpdateAuthorizedViewRequest; +import com.google.bigtable.admin.v2.UpdateLogicalViewMetadata; +import com.google.bigtable.admin.v2.UpdateLogicalViewRequest; +import com.google.bigtable.admin.v2.UpdateMaterializedViewMetadata; +import com.google.bigtable.admin.v2.UpdateMaterializedViewRequest; +import com.google.bigtable.admin.v2.UpdateTableMetadata; +import com.google.bigtable.admin.v2.UpdateTableRequest; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListAppProfilesPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListAppProfilesPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListLogicalViewsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListLogicalViewsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListMaterializedViewsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListMaterializedViewsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListAuthorizedViewsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListAuthorizedViewsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListBackupsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListBackupsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListTablesPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListTablesPagedResponse; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.internal.NameUtil; +import com.google.cloud.bigtable.admin.v2.stub.BigtableInstanceAdminStub; +import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub; +import com.google.cloud.bigtable.admin.v2.stub.GrpcBigtableTableAdminStub; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Duration; +import com.google.protobuf.Empty; +import com.google.protobuf.FieldMask; +import io.grpc.Status.Code; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +class StaleResourceCleanerTest { + + private static final String PROJECT_ID = "test-project"; + private static final String SHARED_INSTANCE_ID = "google-cloud-bigtable"; + private static final String STALE_PREFIX = "temp-0850000000"; + private static final String STALE_ID_PREFIX = "temp-0800000001-"; + private static final String FRESH_ID_PREFIX = "temp-0899999999-"; + + private BigtableInstanceAdminStub mockInstanceStub; + private BigtableInstanceAdminClient instanceAdmin; + private Map tableStubsByInstance; + + private UnaryCallable + mockListMaterializedViewsCallable; + private OperationCallable< + UpdateMaterializedViewRequest, MaterializedView, UpdateMaterializedViewMetadata> + mockUpdateMaterializedViewCallable; + private UnaryCallable mockDeleteMaterializedViewCallable; + + private UnaryCallable + mockListLogicalViewsCallable; + private OperationCallable + mockUpdateLogicalViewCallable; + private UnaryCallable mockDeleteLogicalViewCallable; + + private UnaryCallable + mockListAppProfilesCallable; + private UnaryCallable mockDeleteAppProfileCallable; + + private UnaryCallable mockListClustersCallable; + private UnaryCallable mockDeleteClusterCallable; + + private UnaryCallable mockListInstancesCallable; + private UnaryCallable mockDeleteInstanceCallable; + + @SuppressWarnings("unchecked") + @BeforeEach + void setUp() { + mockInstanceStub = mock(BigtableInstanceAdminStub.class, withSettings().withoutAnnotations()); + instanceAdmin = BigtableInstanceAdminClient.create(PROJECT_ID, mockInstanceStub); + tableStubsByInstance = new HashMap<>(); + + mockListMaterializedViewsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockUpdateMaterializedViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + mockDeleteMaterializedViewCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListLogicalViewsCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockUpdateLogicalViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + mockDeleteLogicalViewCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListAppProfilesCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockDeleteAppProfileCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListClustersCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockDeleteClusterCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListInstancesCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockDeleteInstanceCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + when(mockInstanceStub.listMaterializedViewsPagedCallable()) + .thenReturn(mockListMaterializedViewsCallable); + when(mockInstanceStub.updateMaterializedViewOperationCallable()) + .thenReturn(mockUpdateMaterializedViewCallable); + when(mockInstanceStub.deleteMaterializedViewCallable()) + .thenReturn(mockDeleteMaterializedViewCallable); + + when(mockInstanceStub.listLogicalViewsPagedCallable()).thenReturn(mockListLogicalViewsCallable); + when(mockInstanceStub.updateLogicalViewOperationCallable()) + .thenReturn(mockUpdateLogicalViewCallable); + when(mockInstanceStub.deleteLogicalViewCallable()).thenReturn(mockDeleteLogicalViewCallable); + + when(mockInstanceStub.listAppProfilesPagedCallable()).thenReturn(mockListAppProfilesCallable); + when(mockInstanceStub.deleteAppProfileCallable()).thenReturn(mockDeleteAppProfileCallable); + + when(mockInstanceStub.listClustersCallable()).thenReturn(mockListClustersCallable); + when(mockInstanceStub.deleteClusterCallable()).thenReturn(mockDeleteClusterCallable); + + when(mockInstanceStub.listInstancesCallable()).thenReturn(mockListInstancesCallable); + when(mockInstanceStub.deleteInstanceCallable()).thenReturn(mockDeleteInstanceCallable); + + when(mockDeleteMaterializedViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteLogicalViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteAppProfileCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteClusterCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteInstanceCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + } + + @Test + @SuppressWarnings("unchecked") + void testCleanUpSharedInstance_viewsTablesAndAuthorizedViews() throws Exception { + EnhancedBigtableTableAdminStub tableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + + String staleProtectedMv = STALE_ID_PREFIX + "mv-protected"; + String staleUnprotectedMv = STALE_ID_PREFIX + "mv-unprotected"; + String freshMv = FRESH_ID_PREFIX + "mv-fresh"; + String permMv = "permanent-mv"; + + mockMaterializedViews( + SHARED_INSTANCE_ID, + ImmutableList.of( + buildMaterializedView(SHARED_INSTANCE_ID, staleProtectedMv, true), + buildMaterializedView(SHARED_INSTANCE_ID, staleUnprotectedMv, false), + buildMaterializedView(SHARED_INSTANCE_ID, freshMv, true), + buildMaterializedView(SHARED_INSTANCE_ID, permMv, false))); + + mockOperationFuture( + mockUpdateMaterializedViewCallable, + buildMaterializedView(SHARED_INSTANCE_ID, staleProtectedMv, false)); + + String staleProtectedLv = STALE_ID_PREFIX + "lv-protected"; + String staleUnprotectedLv = STALE_ID_PREFIX + "lv-unprotected"; + String freshLv = FRESH_ID_PREFIX + "lv-fresh"; + + mockLogicalViews( + SHARED_INSTANCE_ID, + ImmutableList.of( + buildLogicalView(SHARED_INSTANCE_ID, staleProtectedLv, true), + buildLogicalView(SHARED_INSTANCE_ID, staleUnprotectedLv, false), + buildLogicalView(SHARED_INSTANCE_ID, freshLv, true))); + + mockOperationFuture( + mockUpdateLogicalViewCallable, + buildLogicalView(SHARED_INSTANCE_ID, staleProtectedLv, false)); + + String staleTable = STALE_ID_PREFIX + "table"; + String freshTable = FRESH_ID_PREFIX + "table"; + String persistentTable = "integration-tests"; + + mockListTables( + tableStub, SHARED_INSTANCE_ID, ImmutableList.of(staleTable, freshTable, persistentTable)); + + UnaryCallable mockGetTableCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + OperationCallable mockUpdateTableCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteTableCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable + mockListAuthorizedViewsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + OperationCallable + mockUpdateAuthorizedViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteAuthorizedViewCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + when(tableStub.getTableCallable()).thenReturn(mockGetTableCallable); + when(tableStub.updateTableOperationCallable()).thenReturn(mockUpdateTableCallable); + when(tableStub.deleteTableCallable()).thenReturn(mockDeleteTableCallable); + when(tableStub.listAuthorizedViewsPagedCallable()).thenReturn(mockListAuthorizedViewsCallable); + when(tableStub.updateAuthorizedViewOperationCallable()) + .thenReturn(mockUpdateAuthorizedViewCallable); + when(tableStub.deleteAuthorizedViewCallable()).thenReturn(mockDeleteAuthorizedViewCallable); + + Table staleTableProto = + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .setDeletionProtection(true) + .setChangeStreamConfig( + ChangeStreamConfig.newBuilder() + .setRetentionPeriod(Duration.newBuilder().setSeconds(86400))) + .build(); + when(mockGetTableCallable.futureCall( + GetTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .setView(View.SCHEMA_VIEW) + .build())) + .thenReturn(ApiFutures.immediateFuture(staleTableProto)); + + mockOperationFuture( + mockUpdateTableCallable, + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .setDeletionProtection(false) + .build()); + when(mockDeleteTableCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + String viewOnStaleTable = "av-on-stale-table"; + mockAuthorizedViews( + mockListAuthorizedViewsCallable, + SHARED_INSTANCE_ID, + staleTable, + ImmutableList.of(viewOnStaleTable)); + + String staleAvOnPersistentTable = STALE_ID_PREFIX + "av-stale"; + String freshAvOnPersistentTable = FRESH_ID_PREFIX + "av-fresh"; + String permAvOnPersistentTable = "perm-av"; + mockAuthorizedViews( + mockListAuthorizedViewsCallable, + SHARED_INSTANCE_ID, + persistentTable, + ImmutableList.of( + staleAvOnPersistentTable, freshAvOnPersistentTable, permAvOnPersistentTable)); + + mockOperationFuture( + mockUpdateAuthorizedViewCallable, + AuthorizedView.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, staleAvOnPersistentTable)) + .setSubsetView(AuthorizedView.SubsetView.getDefaultInstance()) + .setDeletionProtection(false) + .build()); + when(mockDeleteAuthorizedViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of()); + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + mockInstances(ImmutableList.of()); + + StaleResourceCleaner cleaner = createCleaner(); + cleaner.cleanUp(STALE_PREFIX); + + // MaterializedViews: stale ones deleted before table deletion; only protected one updated + InOrder inOrder = + inOrder( + mockUpdateMaterializedViewCallable, + mockDeleteMaterializedViewCallable, + mockUpdateLogicalViewCallable, + mockDeleteLogicalViewCallable, + mockDeleteTableCallable); + inOrder + .verify(mockUpdateMaterializedViewCallable) + .futureCall( + UpdateMaterializedViewRequest.newBuilder() + .setMaterializedView( + MaterializedView.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedMv)) + .setDeletionProtection(false)) + .setUpdateMask(FieldMask.newBuilder().addPaths("deletion_protection")) + .build()); + inOrder + .verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedMv)) + .build()); + inOrder + .verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleUnprotectedMv)) + .build()); + inOrder + .verify(mockUpdateLogicalViewCallable) + .futureCall( + UpdateLogicalViewRequest.newBuilder() + .setLogicalView( + LogicalView.newBuilder() + .setName( + NameUtil.formatLogicalViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedLv)) + .setDeletionProtection(false)) + .setUpdateMask(FieldMask.newBuilder().addPaths("deletion_protection")) + .build()); + inOrder + .verify(mockDeleteLogicalViewCallable) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName( + NameUtil.formatLogicalViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedLv)) + .build()); + inOrder + .verify(mockDeleteLogicalViewCallable) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName( + NameUtil.formatLogicalViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleUnprotectedLv)) + .build()); + inOrder + .verify(mockDeleteTableCallable) + .futureCall( + DeleteTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .build()); + + verify(mockDeleteMaterializedViewCallable, never()) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, SHARED_INSTANCE_ID, freshMv)) + .build()); + verify(mockDeleteMaterializedViewCallable, never()) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, SHARED_INSTANCE_ID, permMv)) + .build()); + verify(mockDeleteLogicalViewCallable, never()) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName(NameUtil.formatLogicalViewName(PROJECT_ID, SHARED_INSTANCE_ID, freshLv)) + .build()); + + // Persistent table is NOT deleted, but its stale AuthorizedView is unprotected and deleted + verify(mockDeleteTableCallable, never()) + .futureCall( + DeleteTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, persistentTable)) + .build()); + verify(mockDeleteTableCallable, never()) + .futureCall( + DeleteTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, freshTable)) + .build()); + + verify(mockDeleteAuthorizedViewCallable) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, staleAvOnPersistentTable)) + .build()); + verify(mockDeleteAuthorizedViewCallable, never()) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, freshAvOnPersistentTable)) + .build()); + verify(mockDeleteAuthorizedViewCallable, never()) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, permAvOnPersistentTable)) + .build()); + } + + @Test + @SuppressWarnings("unchecked") + void testCleanUpSharedInstance_appProfilesClustersAndBackups() throws Exception { + EnhancedBigtableTableAdminStub tableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + + mockMaterializedViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockListTables(tableStub, SHARED_INSTANCE_ID, ImmutableList.of()); + + String staleAp = STALE_ID_PREFIX + "ap"; + String freshAp = FRESH_ID_PREFIX + "ap"; + String defaultAp = "default"; + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of(staleAp, freshAp, defaultAp)); + + String primaryCluster = "google-cloud-bigtable-c0"; + String staleCluster = STALE_ID_PREFIX + "cluster"; + String freshCluster = FRESH_ID_PREFIX + "cluster"; + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of(primaryCluster, staleCluster, freshCluster)); + + UnaryCallable mockListBackupsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteBackupCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(tableStub.listBackupsPagedCallable()).thenReturn(mockListBackupsCallable); + when(tableStub.deleteBackupCallable()).thenReturn(mockDeleteBackupCallable); + when(mockDeleteBackupCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + String staleBackupInPrimary = STALE_ID_PREFIX + "backup"; + String freshBackupInPrimary = FRESH_ID_PREFIX + "backup"; + String permBackupInPrimary = "perm-backup"; + mockBackups( + mockListBackupsCallable, + SHARED_INSTANCE_ID, + primaryCluster, + ImmutableList.of(staleBackupInPrimary, freshBackupInPrimary, permBackupInPrimary)); + + String backupInStaleCluster = "any-backup-id"; + mockBackups( + mockListBackupsCallable, + SHARED_INSTANCE_ID, + staleCluster, + ImmutableList.of(backupInStaleCluster)); + + mockInstances(ImmutableList.of()); + + StaleResourceCleaner cleaner = createCleaner(); + cleaner.cleanUp(STALE_PREFIX); + + // AppProfiles: only staleAp deleted with ignoreWarnings=true + verify(mockDeleteAppProfileCallable) + .futureCall( + DeleteAppProfileRequest.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, staleAp)) + .setIgnoreWarnings(true) + .build()); + verify(mockDeleteAppProfileCallable, never()) + .futureCall( + DeleteAppProfileRequest.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, freshAp)) + .setIgnoreWarnings(true) + .build()); + + // Primary cluster: not deleted, only stale backup in it is deleted + verify(mockDeleteClusterCallable, never()) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster)) + .build()); + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster, staleBackupInPrimary)) + .build()); + verify(mockDeleteBackupCallable, never()) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster, freshBackupInPrimary)) + .build()); + verify(mockDeleteBackupCallable, never()) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster, permBackupInPrimary)) + .build()); + + // Stale cluster: all backups deleted, cluster deleted + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, staleCluster, backupInStaleCluster)) + .build()); + verify(mockDeleteClusterCallable) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, SHARED_INSTANCE_ID, staleCluster)) + .build()); + } + + @Test + @SuppressWarnings("unchecked") + void testCleanUpStaleInstances() throws Exception { + String staleInstance = STALE_ID_PREFIX + "inst"; + String freshInstance = FRESH_ID_PREFIX + "inst"; + + mockInstances(ImmutableList.of(SHARED_INSTANCE_ID, staleInstance, freshInstance)); + + String mvOnStaleInst = "mv-1"; + String lvOnStaleInst = "lv-1"; + mockMaterializedViews( + staleInstance, ImmutableList.of(buildMaterializedView(staleInstance, mvOnStaleInst, true))); + mockLogicalViews( + staleInstance, ImmutableList.of(buildLogicalView(staleInstance, lvOnStaleInst, true))); + mockOperationFuture( + mockUpdateMaterializedViewCallable, + buildMaterializedView(staleInstance, mvOnStaleInst, false)); + mockOperationFuture( + mockUpdateLogicalViewCallable, buildLogicalView(staleInstance, lvOnStaleInst, false)); + + String cluster0 = "c0"; + String cluster1 = "c1"; + mockClusters(staleInstance, ImmutableList.of(cluster0, cluster1)); + + EnhancedBigtableTableAdminStub staleInstTableStub = createTableStubForInstance(staleInstance); + UnaryCallable mockListBackupsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteBackupCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(staleInstTableStub.listBackupsPagedCallable()).thenReturn(mockListBackupsCallable); + when(staleInstTableStub.deleteBackupCallable()).thenReturn(mockDeleteBackupCallable); + when(mockDeleteBackupCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + mockBackups(mockListBackupsCallable, staleInstance, cluster0, ImmutableList.of("b0")); + mockBackups(mockListBackupsCallable, staleInstance, cluster1, ImmutableList.of("b1")); + + EnhancedBigtableTableAdminStub sharedTableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + mockMaterializedViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockListTables(sharedTableStub, SHARED_INSTANCE_ID, ImmutableList.of()); + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of()); + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + + StaleResourceCleaner cleaner = createCleaner(); + cleaner.cleanUp(STALE_PREFIX); + + verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, staleInstance, mvOnStaleInst)) + .build()); + verify(mockDeleteLogicalViewCallable) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName(NameUtil.formatLogicalViewName(PROJECT_ID, staleInstance, lvOnStaleInst)) + .build()); + + // Backups in both clusters deleted, only secondary cluster1 deleted before deleteInstance + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, staleInstance, cluster0, "b0")) + .build()); + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, staleInstance, cluster1, "b1")) + .build()); + verify(mockDeleteClusterCallable, never()) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, staleInstance, cluster0)) + .build()); + verify(mockDeleteClusterCallable) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, staleInstance, cluster1)) + .build()); + verify(mockDeleteInstanceCallable) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, staleInstance)) + .build()); + verify(mockDeleteInstanceCallable, never()) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, SHARED_INSTANCE_ID)) + .build()); + verify(mockDeleteInstanceCallable, never()) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, freshInstance)) + .build()); + } + + @Test + @SuppressWarnings("unchecked") + void testConcurrentNotFoundExceptionsAreIgnored() throws Exception { + EnhancedBigtableTableAdminStub sharedTableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + mockMaterializedViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of()); + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + + String concurrentlyDeletedTable = "deleted-persistent-table"; + String survivingPersistentTable = "integration-tests"; + String staleAv = STALE_ID_PREFIX + "av"; + mockListTables( + sharedTableStub, + SHARED_INSTANCE_ID, + ImmutableList.of(concurrentlyDeletedTable, survivingPersistentTable)); + + UnaryCallable + mockListAuthorizedViewsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + OperationCallable + mockUpdateAuthorizedViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteAuthorizedViewCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(sharedTableStub.listAuthorizedViewsPagedCallable()) + .thenReturn(mockListAuthorizedViewsCallable); + when(sharedTableStub.updateAuthorizedViewOperationCallable()) + .thenReturn(mockUpdateAuthorizedViewCallable); + when(sharedTableStub.deleteAuthorizedViewCallable()) + .thenReturn(mockDeleteAuthorizedViewCallable); + + NotFoundException notFound = + new NotFoundException(null, GrpcStatusCode.of(Code.NOT_FOUND), false); + when(mockListAuthorizedViewsCallable.futureCall( + ListAuthorizedViewsRequest.newBuilder() + .setParent( + NameUtil.formatTableName( + PROJECT_ID, SHARED_INSTANCE_ID, concurrentlyDeletedTable)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(notFound)); + mockAuthorizedViews( + mockListAuthorizedViewsCallable, + SHARED_INSTANCE_ID, + survivingPersistentTable, + ImmutableList.of(staleAv)); + mockOperationFuture( + mockUpdateAuthorizedViewCallable, + AuthorizedView.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, survivingPersistentTable, staleAv)) + .setSubsetView(AuthorizedView.SubsetView.getDefaultInstance()) + .setDeletionProtection(false) + .build()); + when(mockDeleteAuthorizedViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + // Stale instances: first one throws NotFoundException in prepInstanceForDelete, second one has + // cluster0 throw NotFoundException in deleteAllBackups but continues to cluster1 and deletes + String concurrentlyDeletedInst = STALE_ID_PREFIX + "deleted-inst"; + String remainingStaleInst = STALE_ID_PREFIX + "remaining-inst"; + mockInstances(ImmutableList.of(concurrentlyDeletedInst, remainingStaleInst)); + + when(mockListMaterializedViewsCallable.futureCall( + ListMaterializedViewsRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, concurrentlyDeletedInst)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(notFound)); + + mockMaterializedViews(remainingStaleInst, ImmutableList.of()); + mockLogicalViews(remainingStaleInst, ImmutableList.of()); + mockClusters(remainingStaleInst, ImmutableList.of("c0", "c1")); + + EnhancedBigtableTableAdminStub remainingInstTableStub = + createTableStubForInstance(remainingStaleInst); + UnaryCallable mockListBackupsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteBackupCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(remainingInstTableStub.listBackupsPagedCallable()).thenReturn(mockListBackupsCallable); + when(remainingInstTableStub.deleteBackupCallable()).thenReturn(mockDeleteBackupCallable); + when(mockDeleteBackupCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + when(mockListBackupsCallable.futureCall( + ListBackupsRequest.newBuilder() + .setParent(NameUtil.formatClusterName(PROJECT_ID, remainingStaleInst, "c0")) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(notFound)); + mockBackups(mockListBackupsCallable, remainingStaleInst, "c1", ImmutableList.of("b1")); + + StaleResourceCleaner cleaner = createCleaner(); + cleaner.cleanUp(STALE_PREFIX); + + verify(mockDeleteAuthorizedViewCallable) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, survivingPersistentTable, staleAv)) + .build()); + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, remainingStaleInst, "c1", "b1")) + .build()); + verify(mockDeleteClusterCallable) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, remainingStaleInst, "c1")) + .build()); + verify(mockDeleteInstanceCallable) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, remainingStaleInst)) + .build()); + } + + @Test + @SuppressWarnings("unchecked") + void testNonNotFoundErrorsContinueCleanupAndThrowAtEnd() throws Exception { + EnhancedBigtableTableAdminStub sharedTableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + InternalException internalError = + new InternalException(null, GrpcStatusCode.of(Code.INTERNAL), false); + + // 1. Materialized views: first stale view throws InternalException on delete, second succeeds + String failingMv = STALE_ID_PREFIX + "mv-fail"; + String succeedingMv = STALE_ID_PREFIX + "mv-ok"; + mockMaterializedViews( + SHARED_INSTANCE_ID, + ImmutableList.of( + buildMaterializedView(SHARED_INSTANCE_ID, failingMv, false), + buildMaterializedView(SHARED_INSTANCE_ID, succeedingMv, false))); + when(mockDeleteMaterializedViewCallable.futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, SHARED_INSTANCE_ID, failingMv)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(internalError)); + + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockListTables(sharedTableStub, SHARED_INSTANCE_ID, ImmutableList.of()); + + // 2. App profiles: first stale app profile throws InternalException on delete, second succeeds + String failingAp = STALE_ID_PREFIX + "ap-fail"; + String succeedingAp = STALE_ID_PREFIX + "ap-ok"; + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of(failingAp, succeedingAp)); + when(mockDeleteAppProfileCallable.futureCall( + DeleteAppProfileRequest.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, failingAp)) + .setIgnoreWarnings(true) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(internalError)); + + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + + // 3. Stale instances: first stale instance fails to delete, second stale instance succeeds + String failingInst = STALE_ID_PREFIX + "inst-fail"; + String succeedingInst = STALE_ID_PREFIX + "inst-ok"; + mockInstances(ImmutableList.of(failingInst, succeedingInst)); + + mockMaterializedViews(failingInst, ImmutableList.of()); + mockLogicalViews(failingInst, ImmutableList.of()); + mockClusters(failingInst, ImmutableList.of()); + createTableStubForInstance(failingInst); + when(mockDeleteInstanceCallable.futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, failingInst)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(internalError)); + + mockMaterializedViews(succeedingInst, ImmutableList.of()); + mockLogicalViews(succeedingInst, ImmutableList.of()); + mockClusters(succeedingInst, ImmutableList.of()); + createTableStubForInstance(succeedingInst); + + StaleResourceCleaner cleaner = createCleaner(); + RuntimeException thrown = + assertThrows(RuntimeException.class, () -> cleaner.cleanUp(STALE_PREFIX)); + + assertThat(thrown.getMessage()).contains("3 error(s)"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getSuppressed()).hasLength(2); + + // Subsequent resources in each phase were still cleaned up despite earlier failures + verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, succeedingMv)) + .build()); + verify(mockDeleteAppProfileCallable) + .futureCall( + DeleteAppProfileRequest.newBuilder() + .setName( + NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, succeedingAp)) + .setIgnoreWarnings(true) + .build()); + verify(mockDeleteInstanceCallable) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, succeedingInst)) + .build()); + } + + private StaleResourceCleaner createCleaner() throws Exception { + AbstractTestEnv mockEnv = mock(AbstractTestEnv.class, withSettings().withoutAnnotations()); + when(mockEnv.getInstanceId()).thenReturn(SHARED_INSTANCE_ID); + when(mockEnv.getInstanceAdminClient()).thenReturn(instanceAdmin); + when(mockEnv.getTableAdminClient()) + .thenReturn( + BigtableTableAdminClient.create( + PROJECT_ID, SHARED_INSTANCE_ID, tableStubsByInstance.get(SHARED_INSTANCE_ID))); + when(mockEnv.getTableAdminClientForInstance(anyString())) + .thenAnswer( + invocation -> { + String instanceId = invocation.getArgument(0); + return BigtableTableAdminClient.create( + PROJECT_ID, instanceId, tableStubsByInstance.get(instanceId)); + }); + return new StaleResourceCleaner(mockEnv); + } + + private EnhancedBigtableTableAdminStub createTableStubForInstance(String instanceId) + throws Exception { + EnhancedBigtableTableAdminStub stub = + mock(EnhancedBigtableTableAdminStub.class, withSettings().withoutAnnotations()); + Field backgroundResourcesField = + GrpcBigtableTableAdminStub.class.getDeclaredField("backgroundResources"); + backgroundResourcesField.setAccessible(true); + backgroundResourcesField.set( + stub, mock(BackgroundResource.class, withSettings().withoutAnnotations())); + tableStubsByInstance.put(instanceId, stub); + return stub; + } + + private MaterializedView buildMaterializedView( + String instanceId, String viewId, boolean deletionProtected) { + return MaterializedView.newBuilder() + .setName(NameUtil.formatMaterializedViewName(PROJECT_ID, instanceId, viewId)) + .setDeletionProtection(deletionProtected) + .setQuery("SELECT 1 FROM table") + .build(); + } + + private LogicalView buildLogicalView( + String instanceId, String viewId, boolean deletionProtected) { + return LogicalView.newBuilder() + .setName(NameUtil.formatLogicalViewName(PROJECT_ID, instanceId, viewId)) + .setDeletionProtection(deletionProtected) + .setQuery("SELECT 1 FROM table") + .build(); + } + + private void mockMaterializedViews(String instanceId, List views) { + ListMaterializedViewsPage page = + mock(ListMaterializedViewsPage.class, withSettings().withoutAnnotations()); + when(page.getValues()).thenReturn(views); + when(page.hasNextPage()).thenReturn(false); + ListMaterializedViewsPagedResponse response = + mock(ListMaterializedViewsPagedResponse.class, withSettings().withoutAnnotations()); + when(response.getPage()).thenReturn(page); + when(mockListMaterializedViewsCallable.futureCall( + ListMaterializedViewsRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockLogicalViews(String instanceId, List views) { + ListLogicalViewsPage page = + mock(ListLogicalViewsPage.class, withSettings().withoutAnnotations()); + when(page.getValues()).thenReturn(views); + when(page.hasNextPage()).thenReturn(false); + ListLogicalViewsPagedResponse response = + mock(ListLogicalViewsPagedResponse.class, withSettings().withoutAnnotations()); + when(response.getPage()).thenReturn(page); + when(mockListLogicalViewsCallable.futureCall( + ListLogicalViewsRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + @SuppressWarnings("unchecked") + private void mockListTables( + EnhancedBigtableTableAdminStub tableStub, String instanceId, List tableIds) { + UnaryCallable mockListTablesCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(tableStub.listTablesPagedCallable()).thenReturn(mockListTablesCallable); + + List tables = new ArrayList<>(); + for (String tableId : tableIds) { + tables.add( + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, instanceId, tableId)) + .build()); + } + ListTablesPage page = mock(ListTablesPage.class, withSettings().withoutAnnotations()); + when(page.getValues()).thenReturn(tables); + when(page.hasNextPage()).thenReturn(false); + ListTablesPagedResponse response = + mock(ListTablesPagedResponse.class, withSettings().withoutAnnotations()); + when(response.getPage()).thenReturn(page); + when(mockListTablesCallable.futureCall( + ListTablesRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockAuthorizedViews( + UnaryCallable callable, + String instanceId, + String tableId, + List viewIds) { + List views = new ArrayList<>(); + for (String viewId : viewIds) { + views.add( + AuthorizedView.newBuilder() + .setName(NameUtil.formatAuthorizedViewName(PROJECT_ID, instanceId, tableId, viewId)) + .build()); + } + ListAuthorizedViewsPage page = + mock(ListAuthorizedViewsPage.class, withSettings().withoutAnnotations()); + when(page.getValues()).thenReturn(views); + when(page.hasNextPage()).thenReturn(false); + ListAuthorizedViewsPagedResponse response = + mock(ListAuthorizedViewsPagedResponse.class, withSettings().withoutAnnotations()); + when(response.getPage()).thenReturn(page); + when(callable.futureCall( + ListAuthorizedViewsRequest.newBuilder() + .setParent(NameUtil.formatTableName(PROJECT_ID, instanceId, tableId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockAppProfiles(String instanceId, List appProfileIds) { + List profiles = new ArrayList<>(); + for (String id : appProfileIds) { + profiles.add( + AppProfile.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, instanceId, id)) + .setMultiClusterRoutingUseAny( + AppProfile.MultiClusterRoutingUseAny.getDefaultInstance()) + .build()); + } + ListAppProfilesPage page = mock(ListAppProfilesPage.class, withSettings().withoutAnnotations()); + when(page.getValues()).thenReturn(profiles); + when(page.hasNextPage()).thenReturn(false); + ListAppProfilesPagedResponse response = + mock(ListAppProfilesPagedResponse.class, withSettings().withoutAnnotations()); + when(response.getPage()).thenReturn(page); + when(mockListAppProfilesCallable.futureCall( + ListAppProfilesRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockClusters(String instanceId, List clusterIds) { + List clusters = new ArrayList<>(); + for (String clusterId : clusterIds) { + clusters.add( + Cluster.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, instanceId, clusterId)) + .setLocation(NameUtil.formatLocationName(PROJECT_ID, "us-central1-b")) + .build()); + } + when(mockListClustersCallable.futureCall( + ListClustersRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn( + ApiFutures.immediateFuture( + ListClustersResponse.newBuilder().addAllClusters(clusters).build())); + } + + private void mockBackups( + UnaryCallable callable, + String instanceId, + String clusterId, + List backupIds) { + List backups = new ArrayList<>(); + for (String backupId : backupIds) { + backups.add( + Backup.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, instanceId, clusterId, backupId)) + .build()); + } + ListBackupsPage page = mock(ListBackupsPage.class, withSettings().withoutAnnotations()); + when(page.getValues()).thenReturn(backups); + when(page.hasNextPage()).thenReturn(false); + ListBackupsPagedResponse response = + mock(ListBackupsPagedResponse.class, withSettings().withoutAnnotations()); + when(response.getPage()).thenReturn(page); + when(callable.futureCall( + ListBackupsRequest.newBuilder() + .setParent(NameUtil.formatClusterName(PROJECT_ID, instanceId, clusterId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockInstances(List instanceIds) { + List instances = new ArrayList<>(); + for (String instanceId : instanceIds) { + instances.add( + Instance.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .setDisplayName(instanceId) + .build()); + } + when(mockListInstancesCallable.futureCall( + ListInstancesRequest.newBuilder() + .setParent(NameUtil.formatProjectName(PROJECT_ID)) + .build())) + .thenReturn( + ApiFutures.immediateFuture( + ListInstancesResponse.newBuilder().addAllInstances(instances).build())); + } + + private void mockOperationFuture( + OperationCallable callable, RespT response) { + OperationSnapshot snapshot = + FakeOperationSnapshot.newBuilder() + .setDone(true) + .setErrorCode(GrpcStatusCode.of(Code.OK)) + .setName("fake-op") + .setResponse(response) + .build(); + OperationFuture future = OperationFutures.immediateOperationFuture(snapshot); + when(callable.futureCall(any())).thenReturn(future); + } +} diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java index dcf0938a5c8b..47d047708597 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java @@ -17,29 +17,6 @@ import static com.google.common.truth.TruthJUnit.assume; -import com.google.api.core.ApiFuture; -import com.google.api.gax.rpc.NotFoundException; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.admin.v2.models.AppProfile; -import com.google.cloud.bigtable.admin.v2.models.Cluster; -import com.google.cloud.bigtable.admin.v2.models.Instance; -import com.google.cloud.bigtable.admin.v2.models.LogicalView; -import com.google.cloud.bigtable.admin.v2.models.MaterializedView; -import com.google.cloud.bigtable.admin.v2.models.Table; -import com.google.cloud.bigtable.admin.v2.models.UpdateAuthorizedViewRequest; -import com.google.cloud.bigtable.admin.v2.models.UpdateLogicalViewRequest; -import com.google.cloud.bigtable.admin.v2.models.UpdateMaterializedViewRequest; -import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; -import com.google.common.collect.ImmutableSet; -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; import org.junit.rules.TestRule; @@ -54,17 +31,11 @@ * *
    *
  • {@code emulator}: uses the cbtemulator component that can be installed by gcloud - *
  • {@code prod}: uses a pre-existing production table. The target table is defined using - * system properties listed in {@link CloudEnv} and application default credentials - *
  • {@code direct_path}: uses a pre-existing table in the direct path test environment. The - * target table is defined using system properties listed in {@link CloudEnv} and application - * default credentials + *
  • {@code cloud}: uses a pre-existing cloud instance and table. The target table is defined + * using system properties listed in {@link CloudEnv} and application default credentials *
* *

By default, {@code emulator} will be used - * - *

Also, when the system property {@code bigtable.grpc-log-dir} is set, it will enable fine - * grained gRPC logging to the configured path. */ public class TestEnvRule implements TestRule { @@ -75,10 +46,6 @@ public class TestEnvRule implements TestRule { private AbstractTestEnv testEnv; - private Handler grpcLogHandler; - private static final Set GRPC_LOGGER_NAMES = - ImmutableSet.of("io.grpc", "io.grpc.netty.shaded"); - @Override public Statement apply(final Statement base, final Description description) { return new Statement() { @@ -121,254 +88,12 @@ protected void before(Description description) throws Throwable { } private void after() { - try { - cleanUpStale(); - } catch (Exception e) { - LOGGER.log(Level.WARNING, "Failed to cleanup environment", e); - } - try { testEnv.stop(); } catch (Exception e) { LOGGER.log(Level.WARNING, "Failed to stop the environment", e); } testEnv = null; - teardownLogging(); - } - - private void teardownLogging() { - if (grpcLogHandler == null) { - return; - } - - for (String grpcLoggerName : GRPC_LOGGER_NAMES) { - Logger.getLogger(grpcLoggerName).removeHandler(grpcLogHandler); - } - grpcLogHandler.flush(); - grpcLogHandler = null; - } - - void cleanUpStale() throws ExecutionException, InterruptedException, IOException { - // Sortable resource prefix - time, process identifier, serial counterck - String stalePrefix = PrefixGenerator.newTimePrefix(Instant.now().minus(1, ChronoUnit.DAYS)); - - cleanupStaleTables(stalePrefix); - if (env().isInstanceAdminSupported()) { - cleanUpStaleAppProfile(stalePrefix); - cleanUpStaleClusters(stalePrefix); - cleanUpStaleInstances(stalePrefix); - } - } - - /** - * Clean up AppProfile that were dynamically created in the default instance that have been - * orphaned. - */ - private void cleanupStaleTables(String stalePrefix) { - LOGGER.info("Start cleaning up stale tables with stalePrefix=" + stalePrefix); - for (String tableId : env().getTableAdminClient().listTables()) { - if (!tableId.startsWith(PrefixGenerator.PREFIX)) { - LOGGER.info("Skip cleaning up table: " + tableId); - continue; - } - if (stalePrefix.compareTo(tableId) > 0) { - LOGGER.info("Preparing stale table for delete: " + tableId); - prepTableForDelete(tableId); - try { - LOGGER.info("Deleting stable table: " + tableId); - env().getTableAdminClient().deleteTable(tableId); - } catch (NotFoundException e) { - LOGGER.log(Level.WARNING, "Deleting stale table failed: " + tableId, e); - } - } - } - } - - private void prepTableForDelete(String tableId) { - if (!(env() instanceof EmulatorEnv)) { - // unprotect table - Table table = env().getTableAdminClient().getTable(tableId); - if (table.isDeletionProtected() || table.getChangeStreamRetention() != null) { - env() - .getTableAdminClient() - .updateTable( - UpdateTableRequest.of(tableId) - .setDeletionProtection(false) - .disableChangeStreamRetention()); - } - - // Unprotected views - for (String viewId : env().getTableAdminClient().listAuthorizedViews(tableId)) { - try { - env() - .getTableAdminClient() - .updateAuthorizedView( - UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the view was already deleted - } - } - } - } - - /** - * Clean up AppProfile that were dynamically created in the default instance that have been - * orphaned. - */ - private void cleanUpStaleAppProfile(String stalePrefix) { - for (AppProfile appProfile : - env().getInstanceAdminClient().listAppProfiles(env().getInstanceId())) { - if (!appProfile.getId().startsWith(PrefixGenerator.PREFIX)) { - continue; - } - boolean isNewerThanStale = appProfile.getId().compareTo(stalePrefix) > 0; - if (isNewerThanStale) { - continue; - } - try { - env() - .getInstanceAdminClient() - .deleteAppProfile(env().getInstanceId(), appProfile.getId(), true); - } catch (NotFoundException ignored) { - // nothing to clean up, the app profile was already deleted - } - } - } - - /** - * Clean up clusters that were dynamically created in the default instance that have been - * orphaned. - */ - private void cleanUpStaleClusters(String stalePrefix) - throws ExecutionException, InterruptedException { - for (Cluster cluster : env().getInstanceAdminClient().listClusters(env().getInstanceId())) { - if (!cluster.getId().startsWith(PrefixGenerator.PREFIX)) { - continue; - } - boolean isNewerThanStale = cluster.getId().compareTo(stalePrefix) > 0; - if (isNewerThanStale) { - continue; - } - - try { - deleteBackups(env().getTableAdminClient(), cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the backup was already deleted - } - try { - env().getInstanceAdminClient().deleteCluster(env().getInstanceId(), cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster was already deleted - } - } - } - - private void prepInstanceForDelete(String instanceId) { - if (env() instanceof EmulatorEnv) { - return; - } - // Unprotected MaterializedViews. - for (MaterializedView materializedView : - env().getInstanceAdminClient().listMaterializedViews(instanceId)) { - try { - env() - .getInstanceAdminClient() - .updateMaterializedView( - UpdateMaterializedViewRequest.of(instanceId, materializedView.getId()) - .setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the materialized view was already deleted - } - } - // Unprotected LogicalViews. - for (LogicalView logicalView : env().getInstanceAdminClient().listLogicalViews(instanceId)) { - try { - env() - .getInstanceAdminClient() - .updateLogicalView( - UpdateLogicalViewRequest.of(instanceId, logicalView.getId()) - .setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the logical view was already deleted - } - } - } - - /** Clean up dynamically created (non-default) instances that have been orphaned. */ - private void cleanUpStaleInstances(String stalePrefix) - throws IOException, ExecutionException, InterruptedException { - for (Instance instance : env().getInstanceAdminClient().listInstances()) { - if (!instance.getId().startsWith(PrefixGenerator.PREFIX)) { - continue; - } - boolean isNewerThanStale = instance.getId().compareTo(stalePrefix) > 0; - if (isNewerThanStale) { - continue; - } - prepInstanceForDelete(instance.getId()); - try { - deleteInstance(instance.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the instance was already deleted - } - } - } - - /** Delete an instance with all of its resources. */ - private void deleteInstance(String instanceId) - throws IOException, ExecutionException, InterruptedException { - BigtableTableAdminSettings settings = - env().getTableAdminSettings().toBuilder().setInstanceId(instanceId).build(); - - // Delete all child resources (backups & clusters) that wont be automatically deleted - try (BigtableTableAdminClient tableAdmin = BigtableTableAdminClient.create(settings)) { - List clusters = env().getInstanceAdminClient().listClusters(instanceId); - - boolean isFirstCluster = true; - - for (Cluster cluster : clusters) { - deleteBackups(tableAdmin, cluster.getId()); - // Skip the first cluster so that it can be delete by deleteInstance (instances can't exist - // without clusters) - if (!isFirstCluster) { - try { - env().getInstanceAdminClient().deleteCluster(instanceId, cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster was already deleted - } - } - isFirstCluster = false; - } - } - - // Delete everything else - try { - env().getInstanceAdminClient().deleteInstance(instanceId); - } catch (NotFoundException ignored) { - // nothing to clean up, the instance was already deleted - } - } - - private void deleteBackups(BigtableTableAdminClient tableAdmin, String clusterId) - throws ExecutionException, InterruptedException { - List> futures = new ArrayList<>(); - - for (String backupId : tableAdmin.listBackups(clusterId)) { - ApiFuture f = tableAdmin.deleteBackupAsync(clusterId, backupId); - futures.add(f); - } - - for (ApiFuture future : futures) { - try { - future.get(); - } catch (ExecutionException e) { - // Ignore not found - if (e.getCause() instanceof NotFoundException) { - continue; - } - throw e; - } - } } public AbstractTestEnv env() {