diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index c949d0882479..9d064269dcde 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -1319,6 +1319,58 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } } + static String maskRbdKey(String message) { + return message == null ? null : message.replaceAll("key=[^:\\s]*", "key=******"); + } + + private Rados connectToCeph(KVMStoragePool pool) throws RadosException { + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + r.confSet("key", pool.getAuthSecret()); + r.confSet("client_mount_timeout", "30"); + r.connect(); + return r; + } + + private boolean rbdImageExists(KVMStoragePool pool, String imageName) throws RadosException, RbdException { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + return Arrays.asList(new Rbd(io).list()).contains(imageName); + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } + + /** + * Removes an RBD image left behind by a failed copy, so that it does not make every later copy fail with "File exists" + */ + private void removeRbdImageQuietly(KVMStoragePool pool, String imageName) { + try { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + Rbd rbd = new Rbd(io); + if (Arrays.asList(rbd.list()).contains(imageName)) { + rbd.remove(imageName); + logger.info("Removed the incomplete RBD image {}/{} left by the failed copy", pool.getSourceDir(), imageName); + } + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } catch (RadosException | RbdException e) { + logger.warn("Failed to remove the incomplete RBD image {}/{} left by the failed copy, remove it manually: {}", pool.getSourceDir(), imageName, e.getMessage()); + } + } + /** * This function copies a physical disk from Secondary Storage to Primary Storage * or from Primary to Primary Storage @@ -1730,12 +1782,21 @@ to support snapshots(backuped) as qcow2 files. */ * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling) */ logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); + String rbdDestPath = destPool.getSourceDir() + "/" + name; + // Only an image created by this copy may be removed again when the copy fails + boolean destCreatedByCopy = false; try { srcFile = new QemuImgFile(sourcePath, sourceFormat); - String rbdDestPath = destPool.getSourceDir() + "/" + name; String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath); destFile = new QemuImgFile(rbdDestFile, destFormat); + if (rbdImageExists(destPool, name)) { + // qemu-img can't convert into an existing image, and it can't be told if it is complete, so don't touch it + logger.error("RBD image {} already exists, it will not be overwritten by the copy of {}", rbdDestPath, srcFile.getFileName()); + return null; + } + destCreatedByCopy = true; + logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); qemu.convert(srcFile, destFile); logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); @@ -1761,15 +1822,23 @@ to support snapshots(backuped) as qcow2 files. */ r.ioCtxDestroy(io); } catch (QemuImgException | LibvirtException e) { String srcFilename = srcFile != null ? srcFile.getFileName() : null; - String destFilename = destFile != null ? destFile.getFileName() : null; - logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage())); + logger.error("Failed to convert from {} to RBD image {} the error was: {}", srcFilename, rbdDestPath, maskRbdKey(e.getMessage())); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } catch (RadosException e) { logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } catch (RbdException e) { logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } } else { /** diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java index 88346abd0176..da3db1e2fa7e 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java @@ -27,6 +27,7 @@ import java.util.UUID; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -176,4 +177,19 @@ public void testUpdateLocalPoolIops_NullResultFromScript() { Mockito.verify(mockPool, never()).setUsedIops(anyLong()); } + + @Test + public void maskRbdKeyHidesTheCephKey() { + String message = "qemu-img: rbd:pool/image:mon_host=10.0.0.1\\:6789:auth_client_required=cephx:id=pool:key=AQCsecret==:rbd_default_format=2: error while converting raw: error rbd create: File exists"; + String masked = LibvirtStorageAdaptor.maskRbdKey(message); + Assert.assertFalse(masked.contains("AQCsecret")); + Assert.assertTrue(masked.contains("key=******:rbd_default_format=2")); + Assert.assertTrue(masked.endsWith("File exists")); + } + + @Test + public void maskRbdKeyKeepsMessagesWithoutKey() { + Assert.assertNull(LibvirtStorageAdaptor.maskRbdKey(null)); + Assert.assertEquals("timeout", LibvirtStorageAdaptor.maskRbdKey("timeout")); + } }