diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index fe61b3c49234a..154fd57192136 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -287,6 +287,48 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1 ---- +== Deleting Snapshot + +You can delete a snapshot using the `control.sh|bat` script. + +The deletion is performed on all *online* server nodes of the cluster. +[NOTE] +==== +The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted. +==== + +[tabs] +-- +tab:Unix[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.sh --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder. +control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots +---- + +tab:Windows[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.bat --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder. +control.bat --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots +---- +-- + +=== Delete operation limitations + +The delete operation is subject to the following limitations: + +* The deletion is rejected if any concurrent snapshot operation (create, restore, check, or a delete with the same name) is + active for the snapshot. +* The operation is irreversible. It cannot be undone, and the deleted snapshot cannot be restored. +* The command prompts for a confirmation before the deletion because the operation is irreversible and cannot be undone. + == Getting Snapshot Operation Status The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface: diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 0fc350ab7c8fc..99882f4ccc6ff 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -81,6 +81,7 @@ import org.apache.ignite.internal.management.performancestatistics.PerformanceStatisticsCommand; import org.apache.ignite.internal.management.property.PropertyCommand; import org.apache.ignite.internal.management.snapshot.SnapshotCommand; +import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand; import org.apache.ignite.internal.management.snapshot.SnapshotRestoreCommand; import org.apache.ignite.internal.management.tx.TxCommand; import org.apache.ignite.internal.management.tx.TxCommandArg; @@ -529,6 +530,8 @@ else if (cmd.getClass() == EncryptionChangeCacheKeyCommand.class) cmdText = F.concat(cmdText, "cacheGroup1"); else if (cmd.getClass() == SnapshotRestoreCommand.class) cmdText = F.concat(cmdText, "snp1"); + else if (cmd.getClass() == SnapshotDeleteCommand.class) + cmdText = F.concat(cmdText, "snp1"); else if (cmd.getClass() == MetaUpdateCommand.class) return; else if (cmd.getClass() == MetaRemoveCommand.class) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java index 6528d919f6105..91a1d67d0dfc0 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java @@ -27,6 +27,7 @@ import org.apache.ignite.util.GridCommandHandlerCheckpointTest; import org.apache.ignite.util.GridCommandHandlerClusterByClassTest; import org.apache.ignite.util.GridCommandHandlerClusterByClassWithSSLTest; +import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest; import org.apache.ignite.util.GridCommandHandlerIncompatibleSslConfigTest; import org.apache.ignite.util.GridCommandHandlerIndexingCheckSizeTest; import org.apache.ignite.util.GridCommandHandlerIndexingClusterByClassTest; @@ -74,6 +75,7 @@ GridCommandHandlerCheckIndexesInlineSizeTest.class, GridCommandHandlerMetadataTest.class, GridCommandHandlerCheckIncrementalSnapshotTest.class, + GridCommandHandlerDeleteSnapshotTest.class, GridCommandHandlerLegacyClientTest.class, KillCommandsControlShTest.class, diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java new file mode 100644 index 0000000000000..967f7b141bf33 --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.util; + +import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import static java.nio.file.Files.newDirectoryStream; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assume.assumeTrue; + +/** Test for the command '--snapshot delete'. */ +@RunWith(Parameterized.class) +public class GridCommandHandlerDeleteSnapshotTest extends GridCommandHandlerAbstractTest { + /** Value: -1 - do not use, 1 - server node, 0 - client node. */ + @Parameter(1) + public int extraNodeIsServer = -1; + + /** */ + @Parameter(2) + public boolean addIncrements; + + /** */ + @Parameter(3) + public boolean changeBaseline; + + /** */ + @Parameter(4) + public boolean customPath; + + /** */ + @Parameter(5) + public boolean separatedWorkDir; + + /** */ + @Parameters(name = "client={0},useExtraNode={1},inc={2},chBaseln={3},cstSnpPath={4},ownWorkDir={5}") + public static Collection parameters() { + return GridTestUtils.cartesianProduct( + commandHandlers(), + F.asList(-1, 1, 0), // Use extra node (do not use at all, server node, client node); + F.asList(false, true), // Add increments to the test snapshot; + F.asList(false, true), // Change baseline; + F.asList(false, true), // Use custom snapshot path; + F.asList(true, false) // Separated (own) work directory. + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + /** Handy if test running is interrupted and {@link #afterTest()} isn't invoked. */ + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + // Also cleans separated snapshot working directories and custom snapshot pacthes. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + if (separatedWorkDir) + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** */ + @Test + public void testSnapshotDelete() throws Exception { + // A custom snapshot path actually puts snapshots in a shared directory. This skews the results when dedicated + // work directories are set. + assumeTrue(!customPath || !separatedWorkDir); + + int entriesCnt = 4000; + int initNodes = 3; + + walCompactionEnabled(addIncrements); + + IgniteEx ig = (IgniteEx)startGridsMultiThreaded(initNodes); + + if (changeBaseline) { + ig.cluster().baselineAutoAdjustEnabled(false); + + ig.cluster().setBaselineTopology(ig.cluster().topologyVersion()); + } + + ig.cluster().state(ACTIVE); + + createCacheAndPreload(ig, entriesCnt); + + File cstSnpsRoot = customPath ? new File(U.defaultWorkDirectory(), "ex_snapshots") : null; + File snpDir = new File(customPath ? cstSnpsRoot : ig.context().pdsFolderResolver().fileTree().snapshotsRoot(), "testSnapshot"); + + snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, false, false) + .get(getTestTimeout()); + + if (addIncrements) { + for (int i = 0; i < 3; ++i) { + int dataIdx = entriesCnt + entriesCnt / 4 * i; + + try (IgniteDataStreamer streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) { + for (int d = dataIdx; d < dataIdx + entriesCnt / 4; ++d) + streamer.addData(i, i); + } + + snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, true, false) + .get(getTestTimeout()); + } + } + + // Optionally restarts with the same servers number, but changed baseline. The snapshot is kept on the same + // previous nodes independently of the baseline. + if (changeBaseline) { + ig.destroyCache(DEFAULT_CACHE_NAME); + awaitPartitionMapExchange(); + + stopAllGrids(); + + ig = (IgniteEx)startGridsMultiThreaded(initNodes - 1); + + ig.cluster().setBaselineTopology(ig.cluster().topologyVersion()); + + startGrid(initNodes - 1); + + assertEquals(initNodes - 1, ig.cluster().currentBaselineTopology().size()); + assertEquals(initNodes, ig.cluster().nodes().size()); + } + + // Optionally adds extra server or client node. + if (extraNodeIsServer == 1) + startGrid(initNodes); + else if (extraNodeIsServer == 0) + startGrid(CLIENT_NODE_NAME_PREFIX); + + injectTestSystemOut(); + + // Tests missing snapshot deletion. + if (customPath) { + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath(), "wrongSnapshot")); + + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath() + "_wrongPath", "testSnapshot")); + } + else + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "wrongSnapshot")); + + String out = testOut.toString(); + + assertFalse(out.contains("Snapshot removed on the following nodes")); + assertFalse(out.contains("the following nodes didn't find any snapshot data, nothing to delete")); + assertTrue(out.contains("Snapshot not found on current server nodes")); + + testOut.reset(); + assertTrue(testOut.toString().isEmpty()); + + if (customPath) { + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath(), "testSnapshot")); + } + else + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "testSnapshot")); + + out = testOut.toString(); + + if (separatedWorkDir) { + // When the nodes use own separated work directory, we expect a strict result. + assertTrue(out.contains("Snapshot removed on the following nodes [cnt=%d]:".formatted(initNodes))); + + if (extraNodeIsServer == 1) + assertTrue(out.contains("the following nodes didn't find any snapshot data, nothing to delete [cnt=1]:")); + else if (extraNodeIsServer == 0) + assertFalse(out.contains("the following nodes didn't find any snapshot data, nothing to delete")); + } + else { + // When nodes use a shared work directory, there is a race for the delete operation. One node can get faster + // than others and remove snapshot completely quickly. The others might not find snapshot files. We can be + // only sure that at least one node removes snapshot. + assertTrue(out.contains("Snapshot removed on the following nodes [cnt=")); + } + + assertFalse(out.contains("Snapshot not found on current server nodes")); + + assertTrue(waitForCondition(() -> !snpDir.exists(), getTestTimeout())); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index f7537f0b20f79..95d28ef5ec2b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -162,6 +162,8 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckPartitionHashesResponse; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcessRequest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckResponse; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult; @@ -437,6 +439,8 @@ public CoreMessagesProvider() { register(SnapshotFilesFailureMessage.class); register(IncrementalSnapshotVerifyResult.class); register(IncrementalSnapshotAwareMessage.class); + register(SnapshotDeleteRequest.class); + register(SnapshotDeleteResponse.class); // [6300 - 6400]: Services messages. Most of them originally come from Discovery. msgIdx = 6300; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java index deb5416b8e28b..8979958bd8fb6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java @@ -28,7 +28,8 @@ public SnapshotCommand() { new SnapshotCancelCommand(), new SnapshotCheckCommand(), new SnapshotRestoreCommand(), - new SnapshotStatusCommand() + new SnapshotStatusCommand(), + new SnapshotDeleteCommand() ); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java new file mode 100644 index 0000000000000..e7262ce26942c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import java.util.Collection; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcess; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Snapshot deletion command. + * + * @see SupportedFeatureRegistry#SNAPSHOT_DELETE_FEATURE + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteCommand extends AbstractSnapshotCommand { + /** {@inheritDoc} */ + @Override public String description() { + return "Deletes snapshot and all its increments from all the online server nodes"; + } + + /** {@inheritDoc} */ + @Override public Class argClass() { + return SnapshotDeleteCommandArg.class; + } + + /** {@inheritDoc} */ + @Override public Class taskClass() { + return SnapshotDeleteTask.class; + } + + /** {@inheritDoc} */ + @Override public void printResult(SnapshotDeleteCommandArg arg, SnapshotDeleteProcessResult res, Consumer printer) { + boolean found = false; + + if (!F.isEmpty(res.uncompletedNodes())) { + found = true; + + printer.accept("WARNING, the following nodes found snapshot data but might not remove it completely " + + nodeIdsStrLst(res.uncompletedNodes())); + + printer.accept(""); + } + + if (!F.isEmpty(res.completedNodes())) { + found = true; + + printer.accept("Snapshot removed on the following nodes " + nodeIdsStrLst(res.completedNodes())); + printer.accept(""); + } + + if (found) { + if (!F.isEmpty(res.emptyNodes())) { + printer.accept("NOTE, the following nodes didn't find any snapshot data, nothing to delete " + + nodeIdsStrLst(res.emptyNodes())); + } + } + else { + if (!F.isEmpty(res.emptyNodes())) + printer.accept("Snapshot not found on current server nodes."); + else + printer.accept("Unknown result."); + } + } + + /** */ + private static String nodeIdsStrLst(Collection uuids) { + return "[cnt=" + uuids.size() + "]: " + uuids.stream().map(UUID::toString).collect(Collectors.joining(", ")); + } + + /** {@inheritDoc} */ + @Override public String confirmationPrompt(SnapshotDeleteCommandArg arg) { + return "This will delete snapshot '" + arg.snapshotName() + + "' and all its increments from all online server nodes." + + U.nl() + U.nl() + + "WARNING: the snapshot integrity, topology and correctness aren't checked." + + " Snapshot data on offline server nodes aren't deleted." + + U.nl() + U.nl() + + "The operation is irreversible."; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java new file mode 100644 index 0000000000000..2b7063de333bb --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.management.api.Argument; +import org.apache.ignite.internal.management.api.Positional; + +/** */ +public class SnapshotDeleteCommandArg extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0; + + /** */ + @Order(0) + @Positional + @Argument(description = "Snapshot name") + String snapshotName; + + /** */ + @Order(1) + @Argument(example = "path", optional = true, + description = "Path to the directory where the snapshot is located. " + + "If not specified, the default configured snapshot directory will be used") + String src; + + /** */ + public String snapshotName() { + return snapshotName; + } + + /** */ + public void snapshotName(String snapshotName) { + this.snapshotName = snapshotName; + } + + /** */ + public String src() { + return src; + } + + /** */ + public void src(String src) { + this.src = src; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java new file mode 100644 index 0000000000000..86bac6972bd2f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; + +/** + * @see IgniteSnapshotManager#deleteSnapshot(String, String) + */ +@GridInternal +public class SnapshotDeleteTask extends VisorOneNodeTask { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorJob job(SnapshotDeleteCommandArg arg) { + return new SnapshotDeleteJob(arg, debug); + } + + /** */ + private static class SnapshotDeleteJob extends SnapshotJob { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Snapshot delete task argument. + * @param debug Flag indicating whether debug information should be printed into node log. + */ + protected SnapshotDeleteJob(SnapshotDeleteCommandArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected SnapshotDeleteProcessResult run(SnapshotDeleteCommandArg arg) { + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + return snpMgr.deleteSnapshot(arg.snapshotName(), arg.src()).get(); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index a9f1f42ca872b..1857aabd1f965 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -34,7 +34,7 @@ /** * Information about partitions of a single node. Sent in response to {@link GridDhtPartitionsSingleRequest} and during * processing partitions exchange future.
- * Has to be completelly restored after receiving from another node. + * Has to be completely restored after receiving from another node. * * @see #afterReceive() */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index c6e25c2baa3ca..a389fb1349cd9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -60,6 +60,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -344,6 +345,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Snapshot validation distributed process. */ private final SnapshotCheckProcess checkSnpProc; + /** Distributed process to delete cluster snapshot. */ + private final SnapshotDeleteProcess deleteSnpProc; + /** Check previously performed snapshot operation and delete uncompleted files if we need. */ private final DistributedProcess endSnpProc; @@ -446,6 +450,8 @@ public IgniteSnapshotManager(GridKernalContext ctx) { checkSnpProc = new SnapshotCheckProcess(ctx); + deleteSnpProc = new SnapshotDeleteProcess(ctx); + // Manage remote snapshots. snpRmtMgr = new SequentialRemoteSnapshotManager(); } @@ -664,6 +670,7 @@ public IgniteSnapshotManager(GridKernalContext ctx) { restoreCacheGrpProc.interrupt(stopErr); checkSnpProc.interrupt(stopErr); + deleteSnpProc.interrupt(stopErr); // Try stop all snapshot processing if not yet. for (AbstractSnapshotFutureTask sctx : locSnpTasks.values()) @@ -701,46 +708,90 @@ public IgniteSnapshotManager(GridKernalContext ctx) { /** * @param snpDir Snapshot dir. */ - public void deleteSnapshot(File snpDir) { + public void deleteLocalSnapshot(File snpDir) { if (!snpDir.exists()) return; if (!snpDir.isDirectory()) return; - deleteSnapshot(new SnapshotFileTree( + var sft = new SnapshotFileTree( cctx.kernalContext(), snpDir.getName(), snpDir.getParent(), ft.folderName(), - pdsSettings.consistentId().toString())); + pdsSettings.consistentId().toString() + ); + + deleteLocalSnapshot(sft); } - /** */ - public void deleteSnapshot(SnapshotFileTree sft) { - try { - U.delete(sft.binaryMeta()); - sft.allStorages().forEach(U::delete); - U.delete(sft.meta()); + /** + * Deletes local shapshot data. + * + * @param sft Snapshot file tree + */ + public boolean deleteLocalSnapshot(SnapshotFileTree sft) { + return deleteLocalSnapshot(sft, null); + } + + /** + * Deletes local shapshot data. + * + * @param sft Snapshot file tree + * @param existsFlag Flag to set {@code true} if any snapshot file or directory was found (existed). If {@code null}, ignored. + * @return {@code True}, if data is found and completely deleted or if no data found; + * {@code False}, if data is found but might not be deleted completely or if was canceled. + */ + public boolean deleteLocalSnapshot(SnapshotFileTree sft, @Nullable AtomicBoolean existsFlag) { + AtomicBoolean res = new AtomicBoolean(true); + + sft.allStorages().forEach(f -> deleteAndCheckExisted(f, res, existsFlag)); + + deleteAndCheckExisted(sft.binaryMeta(), res, existsFlag); + deleteAndCheckExisted(sft.binaryMetaRoot(), res, existsFlag); + deleteAndCheckExisted(sft.marshaller(), res, existsFlag); + deleteAndCheckExisted(sft.root(), res, existsFlag); + + return res.get(); + } + + /** + * Deletes file/directory and sets existence and deletion failure flags. + * + * @param f File/directory to delete. If {@code null}, does nothing. + * @param failRes Is set to {@code False} if at least one existed file or directory was denied to delete. + * @param existsFlag Is set to {@code True} if not {@code null} and if at least one file or directory was found (existed). + */ + private void deleteAndCheckExisted(@Nullable File f, AtomicBoolean failRes, @Nullable AtomicBoolean existsFlag) { + if (f == null) + return; + + boolean existed = f.exists(); - deleteDirectory(sft.binaryMetaRoot()); - deleteDirectory(sft.marshaller()); + if (existed && existsFlag != null) + existsFlag.set(true); - // Delete parent dir which is {snapshot_root}/db if empty. - sft.marshaller().getParentFile().delete(); - // Delete root dir which is {snapshot_root} if empty. - sft.root().delete(); + try { + // Additionally checks the existence for the case of concurrent deletion. + if (existed && ((!f.isDirectory() && !U.delete(f)) || (f.isDirectory() && !deleteDirectory(f))) && f.exists()) + failRes.set(false); } catch (IOException e) { - throw new IgniteException(e); + failRes.set(false); } } /** Concurrently traverse the directory and delete all files. */ - private void deleteDirectory(File dir) throws IOException { - Files.walkFileTree(dir.toPath(), new SimpleFileVisitor() { + private boolean deleteDirectory(File dir) throws IOException { + AtomicBoolean res = new AtomicBoolean(true); + + Files.walkFileTree(dir.toPath(), new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - U.delete(file); + var f = file.toFile(); + + if (f.exists() && !U.delete(file) && f.exists()) + res.set(false); return FileVisitResult.CONTINUE; } @@ -751,7 +802,10 @@ private void deleteDirectory(File dir) throws IOException { } @Override public FileVisitResult postVisitDirectory(Path dir, IOException e) { - dir.toFile().delete(); + var f = dir.toFile(); + + if (f.exists() && !f.delete() && f.exists()) + res.set(false); if (log.isInfoEnabled() && e != null) log.info("Snapshot directory cleaned with an exception [dir=" + dir + ", e=" + e.getMessage() + ']'); @@ -759,6 +813,8 @@ private void deleteDirectory(File dir) throws IOException { return FileVisitResult.CONTINUE; } }); + + return res.get(); } /** @@ -813,6 +869,11 @@ private IgniteInternalFuture initLocalSnapshotStartSt "re-encryption process is not finished yet.")); } + if (cctx.snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Snapshot " + + "'%s' is being deleted.".formatted(req.snapshotName()))); + } + List grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId)); Collection comprGrpIds = F.view(grpIds, i -> { CacheGroupDescriptor desc = cctx.cache().cacheGroupDescriptor(i); @@ -1282,7 +1343,7 @@ private IgniteInternalFuture initLocalSnapshotEndStag if (snpStartReq.incremental()) U.delete(snpOp.snapshotFileTree().incrementalSnapshotFileTree(snpStartReq.incrementIndex()).root()); else - deleteSnapshot(snpOp.snapshotFileTree()); + deleteLocalSnapshot(snpOp.snapshotFileTree()); } else if (!F.isEmpty(endReq.warnings())) { // Pass the warnings further to the next stage for the case when snapshot started from not coordinator. @@ -1446,6 +1507,27 @@ public boolean isSnapshotChecking(String snpName) { return checkSnpProc.isSnapshotChecking(snpName); } + /** + * @return {@code True} if a snapshot {@code snpName} delete operation is in progress. + */ + public boolean isSnapshotDeleting(String snpName, @Nullable String snpPath) { + return deleteSnpProc.isSnapshotDeleting(snpName, snpPath); + } + + /** + * Deletes the cluster-wide snapshot with the given name. + *

+ * The operation is rejected if a concurrent snapshot operation (create, restore, check, etc...) is in progress + * for the snapshot. + * + * @param name Snapshot name. + * @param snpPath Snapshot directory path. If {@code null}, the default configured snapshot directory will be used. + * @return Future which will be completed when the snapshot is deleted on all the baseline nodes. + */ + public IgniteFuture deleteSnapshot(String name, @Nullable String snpPath) { + return deleteSnpProc.start(name, snpPath); + } + /** * Sets the streamer warning flag to current snapshot process if it is active. */ @@ -2032,7 +2114,7 @@ public IgniteFutureImpl createSnapshot( if (!incremental && snpExists) { throw new IgniteException("Create snapshot request has been rejected. " + - "Snapshot with given name already exists on local node."); + "Snapshot with given name already exists on local node or the path is not empty."); } if (incremental) { @@ -2263,7 +2345,7 @@ public IgniteFutureImpl restoreSnapshot( if (SnapshotFileTree.incrementSnapshotDir(snpDir)) U.delete(snpDir); else - deleteSnapshot(snpDir); + deleteLocalSnapshot(snpDir); } if (log.isInfoEnabled()) { @@ -3957,7 +4039,7 @@ public LocalSnapshotSender(SnapshotFileTree sft) { log.info("The Local snapshot sender closed. All resources released [dbNodeSnpDir=" + sft.nodeStorage() + ']'); } else { - deleteSnapshot(sft); + deleteLocalSnapshot(sft); if (log.isDebugEnabled()) log.debug("Local snapshot sender closed due to an error occurred: " + th.getMessage()); @@ -4317,7 +4399,7 @@ public CancelSnapshotCallable(UUID reqId, String snpName) { } /** {@inheritDoc} */ - @Override public Boolean call() throws Exception { + @Override public Boolean call() { if (reqId != null) return ignite.context().cache().context().snapshotMgr().cancelLocalSnapshotOperations(reqId); else { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 2f19878617a15..1337ff8541e99 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; @@ -491,8 +492,13 @@ private IgniteInternalFuture prepareAndCheckMetas(UUID ig } if (!ctx.req.requestId().equals(req.requestId())) { - return new GridFinishedFuture<>(new IllegalStateException("Validation of snapshot '" + req.snapshotName() - + "' has already started [ctx=" + ctx + ']')); + return new GridFinishedFuture<>(new IgniteIllegalStateException("Validation of snapshot '" + req.snapshotName() + + "' has already started [req=" + req + ']')); + } + + if (kctx.cache().context().snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Snapshot '" + req.snapshotName() + + "' is being deleted [req=" + req + ']')); } // Excludes non-baseline initiator. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java new file mode 100644 index 0000000000000..7bbd13cf20b8c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.ArrayList; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgniteFuture; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT; + +/** + * Distributed process to delete a cluster snapshot. The operation is rejected if any concurrent snapshot operation is + * active. + */ +public class SnapshotDeleteProcess { + /** Reject operation messages. */ + private static final String OP_REJECT_MSG = "Snapshot deletion was rejected."; + + /** Kernal context. */ + private final GridKernalContext kctx; + + /** Logger. */ + private final IgniteLogger log; + + /** */ + private volatile boolean interrupted; + + /** Cluster-wide operation futures per request id on certain node. */ + private final Map> clusterOpFuts = new ConcurrentHashMap<>(); + + /** Process requests per snapshot name on each server node. */ + private final Set requests = ConcurrentHashMap.newKeySet(); + + /** The distributed process. */ + private final DistributedProcess distrProc; + + /** + * @param ctx Kernal context. + */ + public SnapshotDeleteProcess(GridKernalContext ctx) { + this.kctx = ctx; + + log = ctx.log(getClass()); + + distrProc = new DistributedProcess<>(ctx, DELETE_SNAPSHOT, this::deletePhase, this::reducePhase); + } + + /** + * Starts the cluster snapshot delete process. + * + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path (optional). + * @return Future that will be completed when the snapshot is deleted. + */ + public IgniteFuture start(String snpName, @Nullable String snpPath) { + UUID reqId = UUID.randomUUID(); + + var clusterOpFut = new GridFutureAdapter(); + + clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId)); + + try { + synchronized (clusterOpFuts) { + if (interrupted || kctx.isStopping()) + throw new NodeStoppingException("Failed to start snapshot delete process: node is stopping."); + + clusterOpFuts.put(reqId, clusterOpFut); + } + + SnapshotDeleteRequest req = new SnapshotDeleteRequest(reqId, snpName, snpPath); + + distrProc.start(reqId, req); + } + catch (Throwable t) { + log.error("Failed to start distributed delete snapshot process [snpName=" + snpName + ", snpPath=" + snpPath + ']', t); + + clusterOpFut.onDone(t); + } + + return new IgniteFutureImpl<>(clusterOpFut); + } + + /** */ + private IgniteInternalFuture deletePhase(UUID ignored, SnapshotDeleteRequest req) { + if (interrupted || kctx.isStopping()) { + return new GridFinishedFuture<>(new NodeStoppingException(OP_REJECT_MSG + + " Node is stopping [req=" + req + ']')); + } + + if (kctx.cluster().get().localNode().isClient()) + return new GridFinishedFuture<>(new SnapshotDeleteResponse(null)); + + kctx.security().authorize(ADMIN_SNAPSHOT); + + IgniteSnapshotManager snpMgr = kctx.cache().context().snapshotMgr(); + + var curCreateRq = snpMgr.currentCreateRequest(); + + if (curCreateRq != null && curCreateRq.snpName.equals(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + " Snapshot with this name is being created [req=" + req + ']')); + } + + if (snpMgr.isRestoring(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + " Snapshot with this name is being restored [req=" + req + ']')); + } + + if (snpMgr.isSnapshotChecking(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + " Snapshot with this name is being checked [req=" + req + ']')); + } + + if (!kctx.rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + " The snapshot deletion feature isn't activated yet [req=" + req + ']')); + } + + try { + if (!requests.add(req)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Deletion of the snapshot has already " + + "started [req=" + req + ']')); + } + + GridFutureAdapter reqLocFut = new GridFutureAdapter<>(); + + kctx.pools().getSnapshotExecutorService().submit(() -> { + try { + AtomicBoolean foundFlag = new AtomicBoolean(); + + boolean deleted = snpMgr.deleteLocalSnapshot(new SnapshotFileTree(kctx, req.snpName, req.snpPath), foundFlag); + + SnapshotDeleteResponse.SnapshotDeleteStatus res; + + if (foundFlag.get()) { + if (deleted && log.isInfoEnabled()) + log.info("Snapshot successfully deleted [req=" + req + ']'); + else if (!deleted) + log.warning("Snapshot deleted not completely [req=" + req + ']'); + + res = deleted + ? SnapshotDeleteResponse.SnapshotDeleteStatus.DELETED + : SnapshotDeleteResponse.SnapshotDeleteStatus.PARTLY_DELETED; + } + else { + if (log.isInfoEnabled()) + log.info("Snapshot not found to delete [req=" + req + ']'); + + res = SnapshotDeleteResponse.SnapshotDeleteStatus.NOT_FOUND; + } + + reqLocFut.onDone(new SnapshotDeleteResponse(res)); + } + finally { + requests.remove(req); + } + }); + + if (log.isInfoEnabled()) + log.info("Deletion of snapshot initialized [req=" + req + ']'); + + return reqLocFut; + } + catch (Throwable t) { + requests.remove(req); + + log.warning("An error occurred during snapshot deletion [req=" + req + ']', t); + + return new GridFinishedFuture<>(t); + } + } + + /** */ + private void reducePhase(UUID reqId, Map results, Map errors) { + var clusterOpFut = clusterOpFuts.get(reqId); + + if (clusterOpFut == null) + return; + + assert clusterOpFut != null; + + try { + var errP = F.isEmpty(errors) ? null : F.first(errors.entrySet()); + + if (errP != null) { + log.warning("Snapshot deletion finished with an error [reqId=" + reqId + ", nodeId=" + + errP.getKey() + ", err='" + errP.getValue().getMessage() + "']", errP.getValue()); + + clusterOpFut.onDone(errP.getValue()); + + return; + } + + var completedNodes = new ArrayList(results.size()); + var uncompletedNodes = new ArrayList(results.size()); + var emptyNodes = new ArrayList(results.size()); + + results.forEach((nodeId, nodeRes) -> { + if (nodeRes.res != null) { + switch (nodeRes.res) { + case NOT_FOUND: + emptyNodes.add(nodeId); + break; + case DELETED: + completedNodes.add(nodeId); + break; + case PARTLY_DELETED: + uncompletedNodes.add(nodeId); + break; + default: + throw new IgniteIllegalStateException("Unknown snapshot deletion node result, [nodeRes=" + + nodeRes + ", nodeId=" + nodeId + ']'); + } + } + }); + + clusterOpFut.onDone(new SnapshotDeleteProcessResult( + completedNodes.isEmpty() ? null : completedNodes, + uncompletedNodes.isEmpty() ? null : uncompletedNodes, + emptyNodes.isEmpty() ? null : emptyNodes + )); + } + catch (Throwable t) { + clusterOpFut.onDone(t); + } + } + + /** */ + public boolean isSnapshotDeleting(String snpName, @Nullable String snpPath) { + return requests.contains(new SnapshotDeleteRequest(null, snpName, snpPath)); + } + + /** + * @param err The interrupt reason. + */ + void interrupt(Throwable err) { + synchronized (clusterOpFuts) { + interrupted = true; + } + + clusterOpFuts.forEach((reqId, clusterOpFut) -> clusterOpFut.onDone(err)); + + clusterOpFuts.clear(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java new file mode 100644 index 0000000000000..95a904dc6d55b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.Collection; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** Result of {@link SnapshotDeleteProcess}. */ +public final class SnapshotDeleteProcessResult extends IgniteDataTransferObject { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Nodes which found snapshot data and completely removed it. */ + @Order(0) + @Nullable Collection completedNodes; + + /** Nodes which found snapshot data but didn't remove it completely. */ + @Order(1) + @Nullable Collection uncompletedNodes; + + /** Server nodes which didn't find any snapshot data. */ + @Order(2) + @Nullable Collection emptyNodes; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteProcessResult() { + // No-op. + } + + /** */ + public SnapshotDeleteProcessResult( + @Nullable Collection completedNodes, + @Nullable Collection uncompletedNodes, + @Nullable Collection emptyNodes + ) { + this.completedNodes = completedNodes; + this.uncompletedNodes = uncompletedNodes; + this.emptyNodes = emptyNodes; + } + + /** */ + public @Nullable Collection completedNodes() { + return completedNodes; + } + + /** */ + public @Nullable Collection uncompletedNodes() { + return uncompletedNodes; + } + + /** */ + public @Nullable Collection emptyNodes() { + return emptyNodes; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java new file mode 100644 index 0000000000000..37577c1b2eb60 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Cluster snapshot delete distributed process request. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteRequest implements Message { + /** Request ID. */ + @Order(0) + UUID reqId; + + /** Snapshot name. */ + @Order(1) + String snpName; + + /** Snapshot directory path. */ + @Order(2) + @Nullable String snpPath; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteRequest() { + // No-op. + } + + /** + * @param reqId Request ID. + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. + */ + SnapshotDeleteRequest(UUID reqId, String snpName, @Nullable String snpPath) { + this.reqId = reqId; + this.snpName = snpName; + this.snpPath = snpPath; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) + return false; + + SnapshotDeleteRequest other = (SnapshotDeleteRequest)o; + + return snpName.equals(other.snpName) && Objects.equals(snpPath, other.snpPath); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(snpName, snpPath); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotDeleteRequest.class, this, super.toString()); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java new file mode 100644 index 0000000000000..adb6c66a096e2 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Single-node result of the snapshot deletion distributed process. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteResponse implements Message { + /** {@code null} for client node. */ + @Order(0) + @Nullable SnapshotDeleteStatus res; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteResponse() { + // No-op. + } + + /** {@code null} for client node. */ + SnapshotDeleteResponse(@Nullable SnapshotDeleteStatus res) { + this.res = res; + } + + /** */ + public enum SnapshotDeleteStatus { + /** Snapshot found and completely deleted. */ + DELETED, + + /** Snapshot found but some files or directories might not be deleted (locked). */ + PARTLY_DELETED, + + /** Snapshot not found. */ + NOT_FOUND + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotDeleteResponse.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 77f56f4d4cc7f..f4fe684eba584 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -659,6 +659,9 @@ private IgniteInternalFuture prepare(UUID igno if (snpMgr.isSnapshotCreating()) throw new IgniteCheckedException(OP_REJECT_MSG + "A cluster snapshot operation is in progress."); + if (snpMgr.isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) + throw new IgniteException(OP_REJECT_MSG + "A snapshot '" + req.snapshotName() + "' delete operation is in progress."); + if (ctx.encryption().isMasterKeyChangeInProgress()) { return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Master key changing " + "process is not finished yet.")); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java index 7b3e55b85d3c4..a14e52331cda0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java @@ -93,4 +93,7 @@ public class SupportedFeatureRegistry { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = new IgniteCoreFeature(1); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java index 3d0ce062f4f75..9c428e0fa748c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java @@ -519,5 +519,10 @@ public enum DistributedProcessType { * Cluster version finalization abort process. */ RU_ABORT_VERSION_FINALIZATION, + + /** + * Delete snapshot procedure. + */ + DELETE_SNAPSHOT } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java index 8857a4ddd656e..206bad2c7a63f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java @@ -36,7 +36,10 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -98,6 +101,8 @@ import org.apache.ignite.lang.IgniteFutureCancelledException; import org.apache.ignite.lang.IgniteFutureTimeoutException; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi; @@ -173,6 +178,9 @@ protected Function valueBuilder() { return valBuilder; } + /** */ + protected @Nullable AbstractTestPluginProvider pluginProvider; + /** Enable encryption of all caches in {@code IgniteConfiguration} before start. */ @Parameterized.Parameter public boolean encryption; @@ -182,7 +190,7 @@ protected Function valueBuilder() { public boolean onlyPrimary; /** Parameters. */ - @Parameterized.Parameters(name = "encryption={0}, onlyPrimay={1}") + @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}") public static Collection params() { List res = new ArrayList<>(); @@ -217,6 +225,9 @@ protected static Collection encryptionParameters() { if (cfg.isClientMode()) return cfg; + if (pluginProvider != null) + cfg.setPluginProviders(pluginProvider); + return cfg.setConsistentId(igniteInstanceName) .setDataStorageConfiguration(new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() @@ -283,6 +294,17 @@ public void afterTestSnapshot() throws Exception { cleanPersistenceDir(); } + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + // Clean all: also separated snapshot working directories and custom snapshot pathes. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + /** * @param evts Events to check. * @throws IgniteInterruptedCheckedException If interrupted. @@ -814,6 +836,81 @@ public static void doSnapshotCancellationTest( assertEquals("Snapshot directory must be empty due to snapshot cancelled", 0, snpDir.list().length); } + /** Tests concurrent snapshot deletion. */ + protected void doTestConcurrentSnapshotDeleteOperation( + ExRunnable prepareCluster, + ExRunnable concurrentOp, + @Nullable Function errValidator, + boolean rerunAtTheEnd + ) throws Exception { + CountDownLatch delProcInitLatch = new CountDownLatch(1); + CountDownLatch delProcProceedLatch = new CountDownLatch(1); + + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public boolean deleteLocalSnapshot(SnapshotFileTree sft, @Nullable AtomicBoolean existsFlag) { + delProcInitLatch.countDown(); + + try { + assertTrue(delProcProceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + + return super.deleteLocalSnapshot(sft, existsFlag); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + prepareCluster.run(); + + var delFut = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null); + + assertTrue(delProcInitLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + try { + concurrentOp.run(); + + if (errValidator != null) + throw new IllegalStateException("Exception is not thrown."); + } + catch (Exception e) { + if (errValidator == null || !errValidator.apply(e)) + throw new IllegalStateException("Unexpected exception: " + e.getMessage(), e); + } + + delProcProceedLatch.countDown(); + + var delRes = delFut.get(getTestTimeout()); + + assertFalse(delRes.completedNodes.isEmpty()); + + if (!rerunAtTheEnd) + return; + + assertThrowsAnyCause( + null, + () -> { + concurrentOp.run(); + + return null; + }, + IllegalArgumentException.class, + "Snapshot does not exists " + ); + } + /** * @param sft Snapshot file tree. * @param parts Collection of pairs group and appropriate cache partition to be snapshot. @@ -994,6 +1091,13 @@ public void waitBlockedSize(int size, long timeout) throws IgniteInterruptedChec } } + /** */ + @FunctionalInterface + protected interface ExRunnable { + /** */ + void run() throws Exception; + } + /** */ protected static class Value { /** */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index 306d44224a4d6..268ff54775232 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -46,6 +46,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.BaselineNode; @@ -124,6 +125,7 @@ import static org.apache.ignite.testframework.GridTestUtils.cartesianProduct; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; /** * Cluster-wide snapshot check procedure tests. @@ -1201,6 +1203,37 @@ public void testConcurrentFullCheckAndFullRestoreDeclined() throws Exception { ); } + /** */ + @Test + public void testConcurrentSnapshotDeleteAndCheckOperations() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> prepareGridsAndSnapshot(4, 3, 1, false), + () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, null).get(), + e -> e.getMessage().contains("Snapshot '%s' is being deleted".formatted(SNAPSHOT_NAME)), + true + ); + } + + /** */ + @Test + public void testConcurrentSnapshotDeleteAndCheckOperationsWithDifferentPath() throws Exception { + // The test uses thread blocking. + assumeTrue(snpThrdPoolSz > 1); + + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + prepareGridsAndSnapshot(4, 3, 1, false); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout()); + }, + () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, snpPath).get(), + null, + false + ); + } + /** Tests that concurrent snapshot full check is declined when the same snapshot is being fully restored (checked). */ @Test public void testConcurrentTheSameSnpFullCheckWhenFullyRestoringDeclined() throws Exception { @@ -1409,8 +1442,8 @@ private void prepareGridsAndSnapshot(int servers, int baseLineCnt, int clients, /** * Tests concurrent snapshot operations related to the snapshot checking. * - * @param originatorOp First snapshot operation on an originator node. - * @param trierOp Second concurrent snapshot operation on a trier node. + * @param firstOp First snapshot operation on an originator node. + * @param secondOp Second concurrent snapshot operation on a trier node. * @param firstDelay First distributed process full message of {@code originatorOp} to delay on the coordinator * to launch {@code trierOp}. * @param secondDelay Second distributed process full message of {@code originatorOp} to delay on the coordinator @@ -1422,8 +1455,8 @@ private void prepareGridsAndSnapshot(int servers, int baseLineCnt, int clients, * @param cleaner If not {@code null}, is executed at the end. */ private void doTestConcurrentSnpCheckOperations( - Supplier> originatorOp, - Supplier> trierOp, + Supplier> firstOp, + Supplier> secondOp, DistributedProcess.DistributedProcessType firstDelay, @Nullable DistributedProcess.DistributedProcessType secondDelay, boolean expectFailure, @@ -1440,17 +1473,17 @@ private void doTestConcurrentSnpCheckOperations( && ((FullMessage)msg).type() == firstDelay.ordinal() && (waitForBothFirstDelays || firstDelayed.compareAndSet(false, true))); - IgniteFuture fut = originatorOp.get(); + IgniteFuture fut = firstOp.get(); discoSpi(grid(0)).waitBlocked(getTestTimeout()); - IgniteFuture fut2 = trierOp.get(); + IgniteFuture fut2 = secondOp.get(); if (expectFailure) { assertThrowsAnyCause( log, fut2::get, - IllegalStateException.class, + IgniteIllegalStateException.class, "Validation of snapshot '" + SNAPSHOT_NAME + "' has already started" ); @@ -1473,7 +1506,7 @@ private void doTestConcurrentSnpCheckOperations( assertThrowsAnyCause( log, fut2::get, - IllegalStateException.class, + IgniteIllegalStateException.class, "Validation of snapshot '" + SNAPSHOT_NAME + "' has already started" ); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java new file mode 100644 index 0000000000000..6c608f25a07a5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; + +import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; + +/** */ +public class IgniteClusterSnapshotDeleteRollingUpgradeTest extends AbstractRollingUpgradeTest { + /** */ + private static final int ALL_GRIDS = 4; + + /** */ + private static final int CLIENTS = 1; + + /** */ + private static final String SNP_NAME = "testSnapshot"; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception { + var cfg = super.getConfiguration(igniteInstanceName, ver); + + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) + ) + ); + + return cfg; + } + + /** */ + @Test + public void testConcurrentUnfinishedRU() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.19.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + int testNodeIx = ALL_GRIDS - CLIENTS - 1; + + createCacheAndSnapshot(testNodeIx); + + ru(grid(testNodeIx)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.19.1"); + } + + spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && + snm.type() == RU_PREPARE_VERSION_FINALIZATION.ordinal()); + + var finalizeFut = GridTestUtils.runAsync(() -> ru(testNodeIx).finalizeClusterVersion()); + + assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); + + ensureSnapshotDeletionFailed(); + + spi(grid(testNodeIx)).stopBlock(); + + assertFalse(spi(grid(testNodeIx)).hasBlockedMessages()); + + finalizeFut.get(getTestTimeout()); + + for (int i = 0; i < ALL_GRIDS; i++) + assertFalse(ru(grid(i)).isVersionUpgradeEnabled()); + + assertFalse(F.isEmpty(snp(1).deleteSnapshot(SNP_NAME, null).get(getTestTimeout()).completedNodes)); + } + + /** */ + @Test + public void testNodeNotSupportingSnapshotDeleteFeature() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.19.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + createCacheAndSnapshot(1); + + ensureSnapshotDeletionFailed(); + + ru(grid(0)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.19.1"); + + ensureSnapshotDeletionFailed(); + } + + ru(grid(1)).finalizeClusterVersion(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertFalse(ru(grid(i)).isVersionUpgradeEnabled()); + + assertFalse(F.isEmpty(snp(i).deleteSnapshot(SNP_NAME, null).get().completedNodes)); + + if (i < ALL_GRIDS - 1) + createSnapshot(i); + } + } + + /** */ + private void createCacheAndSnapshot(int gridIdx) { + int partsCnt = 32; + int keysCnt = partsCnt * 10; + + grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.REPLICATED) + .setBackups(1) + .setAffinity(new RendezvousAffinityFunction().setPartitions(32)) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + try (var ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < keysCnt; i++) + ds.addData(i, i); + } + + createSnapshot(gridIdx); + } + + /** */ + private void createSnapshot(int gridIdx) { + snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + } + + /** */ + private void ensureSnapshotDeletionFailed() { + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + assertThrowsAnyCause( + null, + () -> snp(i0).deleteSnapshot(SNP_NAME, null).get(), + IgniteIllegalStateException.class, + "The snapshot deletion feature isn't activated yet" + ); + } + } + + /** */ + private IgniteSnapshotManager snp(int gridIdx) { + return grid(gridIdx).context().cache().context().snapshotMgr(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java new file mode 100644 index 0000000000000..f1c840883b98f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java @@ -0,0 +1,576 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; + +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PRELOAD; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_INCREMENTAL_SNAPSHOT_START; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.START_SNAPSHOT; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +/** */ +@RunWith(Parameterized.class) +public class IgniteClusterSnapshotDeleteTest extends AbstractSnapshotSelfTest { + /** */ + private boolean separatedWorkDir; + + /** */ + @Parameter(2) + public boolean incremental = true; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + var cfg = super.getConfiguration(igniteInstanceName); + + if (separatedWorkDir) + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** Parameters. */ + @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}, incremental={2}") + public static Collection runParams() { + Collection res = new ArrayList<>(); + + for (boolean incremental : F.asList(false, true)) { + for (Object[] src0 : params()) { + Object[] res0 = new Object[src0.length + 1]; + System.arraycopy(src0, 0, res0, 0, src0.length); + + res0[src0.length] = incremental; + + res.add(res0); + } + } + + return res; + } + + /** {@inheritDoc} */ + @Override public void afterTestSnapshot() throws Exception { + super.afterTestSnapshot(); + + G.allGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override public void beforeTestSnapshot() throws Exception { + super.beforeTestSnapshot(); + + /** Handy if test running is interrupted and {@link #afterTestSnapshot()} isn't invoked. */ + cleanPersistenceDir(); + } + + /** Tests snapshot deletion when one node finds snapshot but fails to delete its data. */ + @Test + public void testUncompletedNodes() throws Exception { + separatedWorkDir = true; + + // Simulates a deletion error on some node. + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public boolean deleteLocalSnapshot(SnapshotFileTree sft, @Nullable AtomicBoolean existsFlag) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) { + existsFlag.set(true); + + return false; + } + + return super.deleteLocalSnapshot(sft, existsFlag); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + var delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertTrue(F.isEmpty(delSnpRes.emptyNodes)); + assertFalse(F.isEmpty(delSnpRes.uncompletedNodes)); + assertTrue(delSnpRes.uncompletedNodes.contains(grid(1).localNode().id())); + } + + /** Tests snapshot deletion when one node has no snapshot data. */ + @Test + public void testEmptyNodes() throws Exception { + separatedWorkDir = true; + + startGridsWithCache(2, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + startGrid(G.allGrids().size()); + + var delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertFalse(F.isEmpty(delSnpRes.emptyNodes)); + assertTrue(delSnpRes.emptyNodes.contains(grid(G.allGrids().size() - 1).localNode().id())); + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + } + + /** Tests snapshot deletion repeat after an offline node restarts. */ + @Test + public void testDeletionRepeatAfterOfflineNodeStarts() throws Exception { + separatedWorkDir = true; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + int stoppedNodeIdx = G.allGrids().size() - 1; + + UUID stoppedNodeId = grid(stoppedNodeIdx).localNode().id(); + + stopGrid(stoppedNodeIdx); + + var delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(2, delSnpRes.completedNodes.size()); + assertFalse(delSnpRes.completedNodes.contains(stoppedNodeId)); + + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + assertTrue(F.isEmpty(delSnpRes.emptyNodes)); + + startGrid(stoppedNodeIdx); + + stoppedNodeId = grid(stoppedNodeIdx).localNode().id(); + + delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, delSnpRes.completedNodes.size()); + assertTrue(delSnpRes.completedNodes.contains(stoppedNodeId)); + + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + assertEquals(2, delSnpRes.emptyNodes.size()); + } + + /** Test snapshot deletion process when one node leaves. */ + @Test + public void testNodeStopsInTheMiddle() throws Exception { + // Incremental snapshots don't support only-primary and encryption mode. + assumeTrue(!incremental || !(onlyPrimary || encryption)); + + separatedWorkDir = true; + + CountDownLatch beginLatch = new CountDownLatch(1); + CountDownLatch proceedLatch = new CountDownLatch(1); + + // Simulates a deletion error on some node. + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public boolean deleteLocalSnapshot(SnapshotFileTree sft, @Nullable AtomicBoolean existsFlag) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) { + beginLatch.countDown(); + + try { + assertTrue(proceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + } + + return super.deleteLocalSnapshot(sft, existsFlag); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + var delFut = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null); + + assertTrue(beginLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + UUID stoppedGridId = grid(1).localNode().id(); + + stopGrid(1); + + proceedLatch.countDown(); + + var delRes = delFut.get(getTestTimeout()); + + assertEquals(2, delRes.completedNodes.size()); + assertFalse(delRes.completedNodes.contains(stoppedGridId)); + + startGrid(1); + + delRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, delRes.completedNodes.size()); + assertTrue(delRes.completedNodes.contains(grid(1).localNode().id())); + } + + /** Tests that a concurrent deletion of a snapshot with the same name but different path is allowed. */ + @Test + public void testConcurrentDeleteOfTheSameSnapshotDifferentPath() throws Exception { + // Incremental snapshots don't support encryption and only-primary mode. + assumeTrue(!incremental || !(encryption || onlyPrimary)); + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(TIMEOUT); + + if (incremental) + addIncrementalSnapshot(null); + + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(snpPath); + + TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi(); + + commSpi1.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage msg0 && msg0.type() == DELETE_SNAPSHOT.ordinal()); + + var delFut0 = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null); + var delFut1 = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, snpPath); + + commSpi1.waitForBlocked(2, getTestTimeout()); + + commSpi1.stopBlock(); + + var delRes0 = delFut0.get(getTestTimeout()); + var delRes1 = delFut1.get(getTestTimeout()); + + assertFalse((delRes0.completedNodes().isEmpty())); + assertFalse(delRes1.completedNodes().isEmpty()); + } + + /** Tests that a concurrent deletion of the same snapshot is declined. */ + @Test + public void testConcurrentDeleteOfTheSameSnapshot() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE, false), + () -> snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()), + e -> e.getMessage().contains("Deletion of the snapshot has already started"), + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot check operation is in progress. */ + @Test + public void testSnapshotDeleteWhenCheckInProgress() throws Exception { + // Incremental snapshots don't support encryption. + assumeTrue(!incremental || !encryption); + + doTestConcurrentSnapshotDelete( + () -> new IgniteFutureImpl<>(snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, null, incremental ? 1 : 0)), + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + null, + "Snapshot with this name is being checked", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot create operation is in progress. */ + @Test + public void testSnapshotDeleteWhenCreateInProgress() throws Exception { + // Incremental snapshots don't support encryption and only-primary mode. + assumeTrue(!incremental || !(encryption || onlyPrimary)); + + doTestConcurrentSnapshotDelete( + () -> snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, incremental, onlyPrimary), + F.asList(START_SNAPSHOT, END_SNAPSHOT), + false, + () -> { + snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + if (incremental) + snp(grid(0)).createSnapshot(SNAPSHOT_NAME).get(getTestTimeout()); + }, + "Snapshot with this name is being created", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore begins. */ + @Test + public void testSnapshotDeleteWhenRestoreBegins() throws Exception { + // Incremental snapshots don't support encryption. + assumeTrue(!incremental || !encryption); + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + "Snapshot with this name is being checked", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore is in progress. */ + @Test + public void testSnapshotDeleteWhenRestoreInProgress() throws Exception { + // Incremental snapshots don't support encryption. + assumeTrue(!incremental || !encryption); + + var restoreMsgs = F.asList( + RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE, + RESTORE_CACHE_GROUP_SNAPSHOT_PRELOAD, + RESTORE_CACHE_GROUP_SNAPSHOT_START + ); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + "Snapshot with this name is being restored", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore is in progress but fails. */ + @Test + public void testSnapshotDeleteWhenRestoreProgressFails() throws Exception { + // An in-the-middle failure won't allow to start restoring the incrementals. + assumeFalse(incremental); + + var restoreMsgs = F.asList(RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + SnapshotFileTree sft = snapshotFileTree(grid(1), SNAPSHOT_NAME); + + String failingFilePath = sft.partitionFile(dfltCacheCfg, primaries[0]).getAbsolutePath() + .replace(sft.nodeStorage().getAbsolutePath(), ""); + + grid(1).context().cache().context().snapshotMgr().ioFactory((file, modes) -> { + FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); + + if (file.getPath().endsWith(failingFilePath)) + throw new RuntimeException("Test exception"); + + return delegate; + }); + }, + "Snapshot with this name is being restored", + true + ); + } + + /** + * @param firstOp First cluster-wide snapshot operation. + * @param msgsToWatch {@link SingleNodeMessage#type()} relating to {@code firstOp} to block on one node. + * @param precreateSnp If {@code true}, creates snapshot after the cluster start. + * @param prepareIteration If not {@code null}, is invoked in the beginning of test iteration at each {@code msgsToWatch}. + * @param concurrentMsgErr Test of failed concurrent to {@code firstOp} delete snapshot operation to watch. + * @param ignoreFirstOpFailure If {@code true}, possible failure of {@code firstOp} is ignored. + */ + protected void doTestConcurrentSnapshotDelete( + Supplier> firstOp, + Collection msgsToWatch, + boolean precreateSnp, + @Nullable Runnable prepareIteration, + String concurrentMsgErr, + boolean ignoreFirstOpFailure + ) throws Exception { + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + if (precreateSnp) { + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(TIMEOUT); + + if (incremental) + addIncrementalSnapshot(null); + } + + TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi(); + + for (var nodeResMsgType : msgsToWatch) { + if (log.isInfoEnabled()) + log.info("Iteration with message-to-wait-for type: " + nodeResMsgType); + + if (prepareIteration != null) + prepareIteration.run(); + + commSpi1.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage msg0 && msg0.type() == nodeResMsgType.ordinal()); + + var firstFut = firstOp.get(); + + commSpi1.waitForBlocked(1, getTestTimeout()); + + assertThrowsAnyCause( + null, + () -> { + snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + return null; + }, + IgniteIllegalStateException.class, + concurrentMsgErr + ); + + commSpi1.stopBlock(); + + if (ignoreFirstOpFailure) { + try { + firstFut.get(getTestTimeout()); + } + catch (Exception e) { + if (log.isDebugEnabled()) + log.debug("The first operation failed but a failure is expected. Failure: " + e.getMessage()); + } + } + else + firstFut.get(getTestTimeout()); + } + } + + /** */ + private void addIncrementalSnapshot(@Nullable String path) { + try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = CACHE_KEYS_RANGE; i < CACHE_KEYS_RANGE + CACHE_KEYS_RANGE / 4; i++) + ds.addData(i, i); + } + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, path, true, onlyPrimary).get(getTestTimeout()); + } + + /** {@inheritDoc} */ + @Override protected void awaitPartitionMapExchange() { + try { + super.awaitPartitionMapExchange(); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 78ec729ec931b..84dd1f57c05d8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -333,6 +333,45 @@ public void testCreateSnapshotDuringRestore() throws Exception { assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE); } + /** Tests that snapshot restore is declined when the same snapshot is being deleted. */ + @Test + public void testConcurrentSnapshotDeleteAndRestoreOperations() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE), + () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null).get(), + e -> e.getMessage().contains("Snapshot '%s' is being deleted".formatted(SNAPSHOT_NAME)), + true + ); + } + + /** */ + @Test + public void testConcurrentSnapshotDeleteAndRestoreOperationsWithDifferentPath() throws Exception { + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithSnapshot(3, CACHE_KEYS_RANGE); + + grid(0).createCache(DEFAULT_CACHE_NAME); + + try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < CACHE_KEYS_RANGE; ++i) + ds.addData(i, i); + } + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, snpPath, null).get(), + null, + false + ); + } + /** * Ensures that the cache doesn't start if one of the baseline nodes fails. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 570761dac227f..1270c77394d93 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -615,6 +615,43 @@ public void testSnapshotExistsException() throws Exception { waitForEvents(EVT_CLUSTER_SNAPSHOT_STARTED, EVT_CLUSTER_SNAPSHOT_FAILED); } + /** + * Tests that snapshot create detects concurrent deletion, or detects still existing snapshot or successfully + * proceeds if snapshot already deleted. + */ + @Test + public void testConcurrentSnapshotDeleteOperation() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(); + }, + () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(), + e -> e.getMessage().contains("Snapshot with given name already exists"), + false + ); + } + + /** + * Tests that a concurrent deletion of a same-named snapshot is allowed if it has a different path. + */ + @Test + public void testConcurrentSnapshotDeleteOperationWithDifferentPath() throws Exception { + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(); + }, + () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(), + null, + false + ); + } + /** @throws Exception If fails. */ @Test public void testClusterSnapshotCleanedOnLeft() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java index 38056daa0b428..c6108f4a28fb8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java @@ -21,4 +21,7 @@ public class TestIgniteReleaseFeatures_2_19_1 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = TestIgniteReleaseFeatures_2_19_0.ROLLING_UPGRADE_FEATURE; + + /** */ + public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java index 5673ed2b71af3..857e317599640 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java @@ -21,6 +21,8 @@ import java.util.Collection; import java.util.List; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckTest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotHandlerTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.DynamicSuite; @@ -48,6 +50,8 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotCheckTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotHandlerTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteRollingUpgradeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index dd6ff4e43caf5..c122892ee1f66 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -269,6 +269,13 @@ This utility can do the following commands: Get the status of the current snapshot operation: control.(sh|bat) --snapshot status + Deletes snapshot and all its increments from all the online server nodes: + control.(sh|bat) --snapshot delete snapshot_name [--src path] + + Parameters: + snapshot_name - Snapshot name. + --src path - Path to the directory where the snapshot is located. If not specified, the default configured snapshot directory will be used. + Change cluster tag to new value: control.(sh|bat) --change-tag newTagValue [--yes] diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index 76f0f8c7e0e37..30a77600c6efa 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -269,6 +269,13 @@ This utility can do the following commands: Get the status of the current snapshot operation: control.(sh|bat) --snapshot status + Deletes snapshot and all its increments from all the online server nodes: + control.(sh|bat) --snapshot delete snapshot_name [--src path] + + Parameters: + snapshot_name - Snapshot name. + --src path - Path to the directory where the snapshot is located. If not specified, the default configured snapshot directory will be used. + Change cluster tag to new value: control.(sh|bat) --change-tag newTagValue [--yes]