IGNITE-29050 Introduce snapshot deletion command

docs/_docs/snapshots/snapshots.adoc

@@ -287,6 +287,48 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach
 control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1
 ----
 
+== Deleting Snapshot
+
+You can delete a snapshot using the `control.sh|bat` script.
+
+The deletion is performed on all *online* server nodes of the cluster.
+[NOTE]
+====
+The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted.
+====
+
+[tabs]
+--
+tab:Unix[]
+[source,shell]
+----
+# Delete the snapshot "snapshot_09062021".
+control.sh --snapshot delete snapshot_09062021
+
+# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder.
+control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots
+----
+
+tab:Windows[]
+[source,shell]
+----
+# Delete the snapshot "snapshot_09062021".
+control.bat --snapshot delete snapshot_09062021
+
+# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder.
+control.bat --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots
+----
+--
+
+=== Delete operation limitations
+
+The delete operation is subject to the following limitations:
+
+* The deletion is rejected if any concurrent snapshot operation (create, restore, check, or a delete with the same name) is
+  active for the snapshot.
+* The operation is irreversible. It cannot be undone, and the deleted snapshot cannot be restored.
+* The command prompts for a confirmation before the deletion because the operation is irreversible and cannot be undone.
+
 == Getting Snapshot Operation Status
 
 The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface:

modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java

@@ -81,6 +81,7 @@
 import org.apache.ignite.internal.management.performancestatistics.PerformanceStatisticsCommand;
 import org.apache.ignite.internal.management.property.PropertyCommand;
 import org.apache.ignite.internal.management.snapshot.SnapshotCommand;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand;
 import org.apache.ignite.internal.management.snapshot.SnapshotRestoreCommand;
 import org.apache.ignite.internal.management.tx.TxCommand;
 import org.apache.ignite.internal.management.tx.TxCommandArg;
@@ -529,6 +530,8 @@ else if (cmd.getClass() == EncryptionChangeCacheKeyCommand.class)
             cmdText = F.concat(cmdText, "cacheGroup1");
         else if (cmd.getClass() == SnapshotRestoreCommand.class)
             cmdText = F.concat(cmdText, "snp1");
+        else if (cmd.getClass() == SnapshotDeleteCommand.class)
+            cmdText = F.concat(cmdText, "snp1");
         else if (cmd.getClass() == MetaUpdateCommand.class)
             return;
         else if (cmd.getClass() == MetaRemoveCommand.class)

modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java

@@ -27,6 +27,7 @@
 import org.apache.ignite.util.GridCommandHandlerCheckpointTest;
 import org.apache.ignite.util.GridCommandHandlerClusterByClassTest;
 import org.apache.ignite.util.GridCommandHandlerClusterByClassWithSSLTest;
+import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest;
 import org.apache.ignite.util.GridCommandHandlerIncompatibleSslConfigTest;
 import org.apache.ignite.util.GridCommandHandlerIndexingCheckSizeTest;
 import org.apache.ignite.util.GridCommandHandlerIndexingClusterByClassTest;
@@ -74,6 +75,7 @@
     GridCommandHandlerCheckIndexesInlineSizeTest.class,
     GridCommandHandlerMetadataTest.class,
     GridCommandHandlerCheckIncrementalSnapshotTest.class,
+    GridCommandHandlerDeleteSnapshotTest.class,
     GridCommandHandlerLegacyClientTest.class,
 
     KillCommandsControlShTest.class,

modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java

@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import static java.nio.file.Files.newDirectoryStream;
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.junit.Assume.assumeTrue;
+
+/** Test for the command '--snapshot delete'. */
+@RunWith(Parameterized.class)
+public class GridCommandHandlerDeleteSnapshotTest extends GridCommandHandlerAbstractTest {
+    /** Value: -1 - do not use, 1 - server node, 0 - client node. */
+    @Parameter(1)
+    public int extraNodeIsServer = -1;
+
+    /** */
+    @Parameter(2)
+    public boolean addIncrements;
+
+    /** */
+    @Parameter(3)
+    public boolean changeBaseline;
+
+    /** */
+    @Parameter(4)
+    public boolean customPath;
+
+    /** */
+    @Parameter(5)
+    public boolean separatedWorkDir;
+
+    /** */
+    @Parameters(name = "client={0},useExtraNode={1},inc={2},chBaseln={3},cstSnpPath={4},ownWorkDir={5}")
+    public static Collection<?> parameters() {
+        return GridTestUtils.cartesianProduct(
+            commandHandlers(),
+            F.asList(-1, 1, 0), // Use extra node (do not use at all, server node, client node);
+            F.asList(false, true), // Add increments to the test snapshot;
+            F.asList(false, true), // Change baseline;
+            F.asList(false, true), // Use custom snapshot path;
+            F.asList(true, false) // Separated (own) work directory.
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        /** Handy if test running is interrupted and {@link #afterTest()} isn't invoked. */
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void cleanPersistenceDir() throws Exception {
+        super.cleanPersistenceDir();
+
+        // Also cleans separated snapshot working directories and custom snapshot pacthes.
+        try (DirectoryStream<Path> files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) {
+            for (Path path : files)
+                U.delete(path);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (separatedWorkDir)
+            cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath());
+
+        return cfg;
+    }
+
+    /** */
+    @Test
+    public void testSnapshotDelete() throws Exception {
+        // A custom snapshot path actually puts snapshots in a shared directory. This skews the results when dedicated
+        // work directories are set.
+        assumeTrue(!customPath || !separatedWorkDir);
+
+        int entriesCnt = 4000;
+        int initNodes = 3;
+
+        walCompactionEnabled(addIncrements);
+
+        IgniteEx ig = (IgniteEx)startGridsMultiThreaded(initNodes);
+
+        if (changeBaseline) {
+            ig.cluster().baselineAutoAdjustEnabled(false);
+
+            ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
+        }
+
+        ig.cluster().state(ACTIVE);
+
+        createCacheAndPreload(ig, entriesCnt);
+
+        File cstSnpsRoot = customPath ? new File(U.defaultWorkDirectory(), "ex_snapshots") : null;
+        File snpDir = new File(customPath ? cstSnpsRoot : ig.context().pdsFolderResolver().fileTree().snapshotsRoot(), "testSnapshot");
+
+        snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, false, false)
+            .get(getTestTimeout());
+
+        if (addIncrements) {
+            for (int i = 0; i < 3; ++i) {
+                int dataIdx = entriesCnt + entriesCnt / 4 * i;
+
+                try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
+                    for (int d = dataIdx; d < dataIdx + entriesCnt / 4; ++d)
+                        streamer.addData(i, i);
+                }
+
+                snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, true, false)
+                    .get(getTestTimeout());
+            }
+        }
+
+        // Optionally restarts with the same servers number, but changed baseline. The snapshot is kept on the same
+        // previous nodes independently of the baseline.
+        if (changeBaseline) {
+            ig.destroyCache(DEFAULT_CACHE_NAME);
+            awaitPartitionMapExchange();
+
+            stopAllGrids();
+
+            ig = (IgniteEx)startGridsMultiThreaded(initNodes - 1);
+
+            ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
+
+            startGrid(initNodes - 1);
+
+            assertEquals(initNodes - 1, ig.cluster().currentBaselineTopology().size());
+            assertEquals(initNodes, ig.cluster().nodes().size());
+        }
+
+        // Optionally adds extra server or client node.
+        if (extraNodeIsServer == 1)
+            startGrid(initNodes);
+        else if (extraNodeIsServer == 0)
+            startGrid(CLIENT_NODE_NAME_PREFIX);
+
+        injectTestSystemOut();
+
+        // Tests missing snapshot deletion.
+        if (customPath) {
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src",
+                cstSnpsRoot.getAbsolutePath(), "wrongSnapshot"));
+
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src",
+                cstSnpsRoot.getAbsolutePath() + "_wrongPath", "testSnapshot"));
+        }
+        else
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "wrongSnapshot"));
+
+        String out = testOut.toString();
+
+        assertFalse(out.contains("Snapshot removed on the following nodes"));
+        assertFalse(out.contains("the following nodes didn't find any snapshot data, nothing to delete"));
+        assertTrue(out.contains("Snapshot not found on current server nodes"));
+
+        testOut.reset();
+        assertTrue(testOut.toString().isEmpty());
+
+        if (customPath) {
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src",
+                cstSnpsRoot.getAbsolutePath(), "testSnapshot"));
+        }
+        else
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "testSnapshot"));
+
+        out = testOut.toString();
+
+        if (separatedWorkDir) {
+            // When the nodes use own separated work directory, we expect a strict result.
+            assertTrue(out.contains("Snapshot removed on the following nodes [cnt=%d]:".formatted(initNodes)));
+
+            if (extraNodeIsServer == 1)
+                assertTrue(out.contains("the following nodes didn't find any snapshot data, nothing to delete [cnt=1]:"));
+            else if (extraNodeIsServer == 0)
+                assertFalse(out.contains("the following nodes didn't find any snapshot data, nothing to delete"));
+        }
+        else {
+            // When nodes use a shared work directory, there is a race for the delete operation. One node can get faster
+            // than others and remove snapshot completely quickly. The others might not find snapshot files. We can be
+            // only sure that at least one node removes snapshot.
+            assertTrue(out.contains("Snapshot removed on the following nodes [cnt="));
+        }
+
+        assertFalse(out.contains("Snapshot not found on current server nodes"));
+
+        assertTrue(waitForCondition(() -> !snpDir.exists(), getTestTimeout()));
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java

@@ -162,6 +162,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckPartitionHashesResponse;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcessRequest;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckResponse;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult;
@@ -437,6 +439,8 @@ public CoreMessagesProvider() {
         register(SnapshotFilesFailureMessage.class);
         register(IncrementalSnapshotVerifyResult.class);
         register(IncrementalSnapshotAwareMessage.class);
+        register(SnapshotDeleteRequest.class);
+        register(SnapshotDeleteResponse.class);
 
         // [6300 - 6400]: Services messages. Most of them originally come from Discovery.
         msgIdx = 6300;

modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java

@@ -28,7 +28,8 @@ public SnapshotCommand() {
             new SnapshotCancelCommand(),
             new SnapshotCheckCommand(),
             new SnapshotRestoreCommand(),
-            new SnapshotStatusCommand()
+            new SnapshotStatusCommand(),
+            new SnapshotDeleteCommand()
         );
     }
 }