diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..c5f68918fd56 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,12 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + +4.23.0.0 > 24.0.0: + * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd + LUKS2 (), for both data disks + and root disks. Encryption is transparent to the guest and reuses the + existing CloudStack volume-encryption passphrase handling, so no + additional key store is required. Note: attaching an encrypted RBD volume + to a running Instance requires libvirt >= 10.1.0; booting an Instance from + an encrypted RBD root disk works on older libvirt. diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index c110e4ca94e1..cca2edd70c6b 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -56,6 +56,7 @@ public static String[] toStrings(Host.Type... types) { String HOST_UEFI_ENABLE = "host.uefi.enable"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + String HOST_RBD_VOLUME_ENCRYPTION = "host.volume.encryption.rbd"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; String HOST_VDDK_SUPPORT = "host.vddk.support"; String HOST_VDDK_LIB_DIR = "vddk.lib.dir"; diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 3511b4e88cb9..275f6d25268d 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -172,7 +172,7 @@ public static enum StoragePoolType { LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR CLVM(true, false, EncryptionSupport.Unsupported), CLVM_NG(true, false, EncryptionSupport.Hypervisor), - RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD + RBD(true, true, EncryptionSupport.Hypervisor), // http://libvirt.org/storage.html#StorageBackendRBD ; encrypted natively by librbd (LUKS2, engine='librbd') SharedMountPoint(true, true, EncryptionSupport.Hypervisor), VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers. diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index db7dcc3fb44f..a32b8dd604e7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -92,7 +92,7 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { @Parameter(name = ApiConstants.DISK_PATH, type = CommandType.STRING, - description = "path of the disk image") + description = "path of the disk image. It is the file name on file based storage pools (NFS, Local, SharedMountPoint), and the image name on RBD storage pools") private String diskPath; @Parameter(name = ApiConstants.IMPORT_SOURCE, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index 50f4b9c1fbe5..db5961ac7f8b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -63,7 +63,7 @@ public class ImportVolumeCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.PATH, type = BaseCmd.CommandType.STRING, required = true, - description = "the path of the volume") + description = "the path of the volume. It is the file name on file based storage pools (NFS, Local, SharedMountPoint), and the image name on RBD storage pools") private String path; @Parameter(name = ApiConstants.NAME, diff --git a/commit-pr-map.md b/commit-pr-map.md new file mode 100644 index 000000000000..c7f154888ed0 --- /dev/null +++ b/commit-pr-map.md @@ -0,0 +1,95 @@ + + +# Commit to apache/cloudstack PR mapping + +Range: `6fd8ddabe73` .. `c101689b6da` on `4.23.0-ceph` (50 commits, oldest first). No commit of the first 49 was changed. + +| # | Commit | PR | Subject | Note | +|---|---|---|---|---| +| 1 | `6fd8ddabe73` | [#11931](https://github.com/apache/cloudstack/pull/11931) | smoke test: support ceph storage pools | | +| 2 | `e913d82adf8` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: fix test failures | | +| 3 | `5bc85a1f04f` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: add volume migration which has been fixed | | +| 4 | `dc919fb6fba` | [#11931](https://github.com/apache/cloudstack/pull/11931), [#14386](https://github.com/apache/cloudstack/pull/14386) | test: support Ceph/RBD primary storage in NAS backup smoke test | shared: in both PRs (NAS backup smoke test for Ceph) | +| 5 | `6879406b9b6` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: fix smoke test failures on xen/vmware | | +| 6 | `685225f4b14` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: sleep 10 seconds before attaching volume in test_events_resource.py | | +| 7 | `c31f0a68efc` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: add more time.sleep(10) | | +| 8 | `b00be0e7fc8` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: add more time.sleep(10) in test_vm_strict_host_tags.py | | +| 9 | `1664f246b87` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: fix test_usage.py | | +| 10 | `ddfcfd52779` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: move time.sleep(10) in test_host_maintenance.py | | +| 11 | `37fd033ab71` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: retry domain deletion in tearDown of test_deploy_vms_in_parallel.py | | +| 12 | `3a7b273a82b` | [#13628](https://github.com/apache/cloudstack/pull/13628) | test: fix tests failures caused by others in drs and host maintenance | | +| 13 | `8890c121855` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: fix test_10_list_volumes for zone-wide storage pools | | +| 14 | `689fb2cac63` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test_host_tags: fix cleanup to handle None host tags | | +| 15 | `854a93feb7c` | [#14307](https://github.com/apache/cloudstack/pull/14307) | package: Add missing openssl and decompression utilities to KVM agent dependencies (PR 14307) | already ends with (PR 14307) | +| 16 | `e64a8d5f69e` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: support Host HA on Ceph RBD primary storage | | +| 17 | `cfdd7fcd402` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: support zone-wide storage pools in test_list_storage_pools | | +| 18 | `27d316ee542` | [#14362](https://github.com/apache/cloudstack/pull/14362) | KVM: allow cluster-wide storage pools for VM deployment from volumes | | +| 19 | `270d76a24c4` | (none yet) | KVM: live volume migration on ceph | live volume migration between Ceph RBD: PR is still todo | +| 20 | `3bb8cc024c9` | (none yet) | smoke tests: exercise online volume migration for KVM on RBD | live volume migration between Ceph RBD: PR is still todo | +| 21 | `3a9e6091e93` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: address review comments on Host HA for Ceph RBD | | +| 22 | `2296fb08d30` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: validate input of the RBD Host HA scripts and the Ceph settings of the pool | | +| 23 | `be60d18dc8b` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: keep the Ceph key out of the logs and process list of the RBD Host HA checks, and harden the parsing | | +| 24 | `6f92c9e8fcc` | [#11931](https://github.com/apache/cloudstack/pull/11931) | smoke test: cover RBD/librbd volume encryption in TestVolumeEncryption | added to PR 11931 on 2026-10-14 | +| 25 | `f08f4eb7768` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: run unmanage/import VM lifecycle test on KVM too | added to PR 11931 on 2026-10-14 | +| 26 | `88b92d6afe8` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: merge the direct download deploy tests into one test | added to PR 11931 on 2026-10-14 | +| 27 | `0771561166f` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvmvmactivity_rbd.sh: do not depend on python3, and tell failures from a DEAD host | | +| 28 | `3f603969bb0` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvmvmactivity_rbd.sh: do not decide on an activity state which could not be saved | | +| 29 | `ced2070ccdb` | [#14386](https://github.com/apache/cloudstack/pull/14386) | KVM: support incremental NAS backups of RBD volumes | | +| 30 | `0c1c243801c` | [#14386](https://github.com/apache/cloudstack/pull/14386) | test: run the NAS backup smoke test on a cloud-init template | | +| 31 | `d970fcbf131` | [#14386](https://github.com/apache/cloudstack/pull/14386) | test: validate a whole backup chain, including across a mid-chain delete | | +| 32 | `700d5900f04` | [#14386](https://github.com/apache/cloudstack/pull/14386) | test: verify the guest of a VM created from a backup, not just that it boots | | +| 33 | `f3b0305e8f0` | [#11931](https://github.com/apache/cloudstack/pull/11931) | test: sync the data to disk before taking the VM snapshot | added to PR 11931 on 2026-10-14 | +| 34 | `937542deebd` | [#14188](https://github.com/apache/cloudstack/pull/14188) | kvm: restore ErrorCode import removed by PR 13763 | per the author: fixes the compile error of #14188 (4.22), whose uses of ErrorCode lost their import when #13763 (main) removed it | +| 35 | `d964898c872` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: rewrite the RBD Host HA scripts in Python, with a faster VM-activity check | | +| 36 | `d08fe694f36` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: choose the script which checks the VM activity by the type of the pool | | +| 37 | `2a3717171c4` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: use only the Python bindings of librados and librbd in the Host HA scripts on Ceph RBD | | +| 38 | `52cfddb59b8` | [#14393](https://github.com/apache/cloudstack/pull/14393) | kvm: remove the partial RBD image when a template copy fails | | +| 39 | `1635bf004e0` | [#13877](https://github.com/apache/cloudstack/pull/13877) | kvm: test the NAS take-backup arguments with StringUtils and CollectionUtils | per the author: follow-up to the incremental NAS backup code from abh1sar's 4.22 backport (no commit title matches) | +| 40 | `a9505139484` | [#14386](https://github.com/apache/cloudstack/pull/14386) | nasbackup: carry the backup method separately from the mode | | +| 41 | `c521e4c7c35` | [#14402](https://github.com/apache/cloudstack/pull/14402) | utils: add RbdUtils for the Ceph monitors of an RBD pool | owner is 14402; also stacked in 14320 and 14386 | +| 42 | `3472c0cece7` | [#14402](https://github.com/apache/cloudstack/pull/14402) | kvm: build every Ceph monitor list with RbdUtils | | +| 43 | `a4fc4945c67` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: build the storage pool's Ceph monitors with RbdUtils | | +| 44 | `91ee2e4cc71` | [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: tell a missing object apart from an unreachable cluster in the RBD HA scripts | | +| 45 | `2dab8b2d86b` | [#14386](https://github.com/apache/cloudstack/pull/14386) | Add ceph-common as dependencies | | +| 46 | `a48a4bcaeef` | [#14386](https://github.com/apache/cloudstack/pull/14386) | kvm: build the Ceph monitors of the restore backup command with RbdUtils | | +| 47 | `320a603b4fd` | [#14386](https://github.com/apache/cloudstack/pull/14386) | backup: name the rbd method in the TakeBackupCommand comment | | +| 48 | `f9b1fb9dbed` | [#14386](https://github.com/apache/cloudstack/pull/14386) | kvm: refuse an RBD restore that would destroy a snapshot we did not take | | +| 49 | `ae74fa267a7` | [#14386](https://github.com/apache/cloudstack/pull/14386) | kvm: drop the cephx key file from a trap in nasbackup.sh | | +| 50 | `c101689b6da` | [#14402](https://github.com/apache/cloudstack/pull/14402) | utils: test the escaped colons of an IPv6 Ceph monitor without a port | owner is 14402 (RbdUtils); the same change is the head commit of the other two PR branches that contain RbdUtils, as `708ba29888d` on #14402, `e9b6dda7d77` on [#14320](https://github.com/apache/cloudstack/pull/14320) and `39cc1041aeb` on [#14386](https://github.com/apache/cloudstack/pull/14386) | + +## PRs + +| PR | Title | Base | Author | +|---|---|---|---| +| [#11931](https://github.com/apache/cloudstack/pull/11931) | smoke test: support ceph storage pools | main | weizhouapache | +| [#13628](https://github.com/apache/cloudstack/pull/13628) | test: fix smoke test failures on xen/vmware | main | weizhouapache | +| [#14307](https://github.com/apache/cloudstack/pull/14307) | package: Add missing openssl and decompression utilities to KVM agent dependencies | main | weizhouapache | +| [#14320](https://github.com/apache/cloudstack/pull/14320) | kvm: support Host HA on Ceph RBD primary storage | main | weizhouapache | +| [#14362](https://github.com/apache/cloudstack/pull/14362) | KVM: allow cluster-wide storage pools for VM deployment from volumes | main | weizhouapache | +| [#14386](https://github.com/apache/cloudstack/pull/14386) | NAS backup: support incremental backups of RBD volumes | main | weizhouapache | +| [#14393](https://github.com/apache/cloudstack/pull/14393) | kvm: remove the partial RBD image when a template copy fails | 4.22 | weizhouapache | +| [#14402](https://github.com/apache/cloudstack/pull/14402) | utils,kvm: build every Ceph monitor list with RbdUtils | main | weizhouapache | +| [#14132](https://github.com/apache/cloudstack/pull/14132) | kvm: delete primary-only RBD snapshots before the volume | 4.22 | weizhouapache | +| [#13877](https://github.com/apache/cloudstack/pull/13877) | backport: Incremental NAS backup support for KVM (#13074) | 4.22 | abh1sar (merged) | +| [#14188](https://github.com/apache/cloudstack/pull/14188) | kvm: look RBD volumes up through librbd instead of refreshing the whole pool | 4.22 | bhouse-nexthop (merged) | + +PR 14132 has no commit in this range. Live volume migration between Ceph RBD has no PR yet (todo). + +Commit 50 was added after the first version of this table. Its three copies were pushed to the heads of PRs 14402, 14320 and 14386. diff --git a/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java index 70375c30a1bb..5559057ecd24 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java @@ -124,6 +124,14 @@ public DataTO getDestData() { return destData; } + public void setSrcData(DataTO srcData) { + this.srcData = srcData; + } + + public void setDestData(DataTO destData) { + this.destData = destData; + } + public void setSrcDetails(Map details) { srcDetails = details; } diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index 972c2eaf7bb4..a11f0d011398 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -41,6 +41,22 @@ public class RestoreBackupCommand extends Command { private VirtualMachine.State vmState; private Integer mountTimeout; + /** + * On-NAS format of the backup files: {@code qcow2} (default, also when null) or {@code rbddiff} + * for Ceph/RBD chains written with {@code rbd export-diff}. Decides both the file extension the + * agent looks for and how the volume is reconstructed. + */ + private String backupFormat; + + /** + * For {@code rbddiff} backups: every backup directory of the chain from its full anchor up to + * and including the backup being restored, ordered root-first. {@code rbd import-diff} streams + * have no backing-file metadata, so the chain cannot be self-describing the way rebased qcow2 + * files are — the agent has to be told which diffs to replay and in which order. Null/empty for + * qcow2 backups. + */ + private List backupChainPaths; + protected RestoreBackupCommand() { super(); } @@ -164,4 +180,20 @@ public Integer getMountTimeout() { public void setMountTimeout(Integer mountTimeout) { this.mountTimeout = mountTimeout; } + + public String getBackupFormat() { + return backupFormat; + } + + public void setBackupFormat(String backupFormat) { + this.backupFormat = backupFormat; + } + + public List getBackupChainPaths() { + return backupChainPaths; + } + + public void setBackupChainPaths(List backupChainPaths) { + this.backupChainPaths = backupChainPaths; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java index 34f8d7b8bcdd..c686363abf1a 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -38,6 +38,7 @@ public class TakeBackupCommand extends Command { // Incremental backup fields (NAS provider; null/empty for legacy full-only callers). private String mode; // "full" or "incremental"; null => legacy behaviour (script default) + private String method; // "qcow2" (bitmaps, push mode), "content" (pull mode) or "rbd" (Ceph export-diff); null => "qcow2" private String bitmapNew; // Checkpoint/bitmap name to create with this backup (timestamp-based) private String bitmapParent; // Incremental: parent bitmap to read changes since @@ -125,6 +126,14 @@ public void setMode(String mode) { this.mode = mode; } + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + public String getBitmapNew() { return bitmapNew; } diff --git a/debian/control b/debian/control index cdf663ef8906..a2117c17b05b 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, python3-rados, python3-rbd, ceph-common, socat, openssl, bzip2, gzip, unzip Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 141596407fe8..8614197347aa 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -183,10 +183,12 @@ List allocateTemplatedVolumes(Type type, String name, DiskOffering */ DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo); + Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, + Storage.ImageFormat format); DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile); + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile, + Storage.ImageFormat format); /** * Unmanage VM volumes diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index 53bfcce27038..f4f8d342eda7 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -33,7 +33,8 @@ */ public interface HighAvailabilityManager extends Manager { - List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint, + StoragePoolType.RBD); ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 2fad96ec1da2..4f163b9a1c9e 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -2626,7 +2626,8 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo, @Override public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, + ImageFormat format) { if (sizeInBytes == null) { sizeInBytes = offering.getDiskSize(); } @@ -2665,7 +2666,9 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(hypervisorType)); + // The format the hypervisor actually reported for the existing image wins; pools such as RBD + // hold raw images even though QCOW2 is the cluster default for KVM. + vol.setFormat(format != null ? format : getSupportedImageFormatForCluster(hypervisorType)); vol.setPoolId(poolId); vol.setPoolType(poolType); vol.setPath(path); @@ -2677,7 +2680,8 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L @Override public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile, + ImageFormat format) { VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId()); if (vm != null) { @@ -2709,7 +2713,9 @@ public DiskProfile updateImportedVolume(Type type, DiskOffering offering, Virtua vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); + // The format the hypervisor actually reported for the existing image wins; pools such as RBD + // hold raw images even though QCOW2 is the cluster default for KVM. + vol.setFormat(format != null ? format : getSupportedImageFormatForCluster(vm.getHypervisorType())); vol.setPoolId(poolId); vol.setPoolType(poolType); vol.setPath(path); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index d61caf16ae5d..8336c5ae5ef7 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -246,7 +246,7 @@ public void testImportVolume() { volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null, zoneId, hypervisorType, null, null, owner, - deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo); + deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo, null); VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0); Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong()); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index bbf775de27ad..be2d1faf2f3e 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -634,6 +634,8 @@ protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) { command.setContextParam(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString()); } + command.setSrcData(srcData.getTO()); + command.setDestData(destData.getTO()); boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData); EndPoint ep = selector.select(srcData, StorageAction.MIGRATEVOLUME); @@ -667,6 +669,23 @@ protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) { volumeVo.setLastPoolId(oldPoolId); if (destPool.getPoolType() == StoragePoolType.CLVM) { volumeVo.setFormat(ImageFormat.RAW); + } else if (destPool.getPoolType() == StoragePoolType.RBD) { + // RBD only ever stores raw images; keep the volume's format metadata consistent + // regardless of what format it had on the source pool (e.g. QCOW2 on NFS). + volumeVo.setFormat(ImageFormat.RAW); + } else if (srcPool.getPoolType() == StoragePoolType.RBD && volumeVo.getFormat() == ImageFormat.RAW + && (destPool.getPoolType() == StoragePoolType.NetworkFilesystem + || destPool.getPoolType() == StoragePoolType.SharedMountPoint + || destPool.getPoolType() == StoragePoolType.Filesystem)) { + // The KVM agent (LibvirtMigrateVolumeCommandWrapper) writes a real QCOW2 file when the + // destination is file-based; without updating this, the volume keeps looking RAW to + // LibvirtComputingResource, which then (mis)treats a plain qcow2 file on NFS as a raw + // block device the next time the Instance is started. + volumeVo.setFormat(ImageFormat.QCOW2); + } + if (volumeVo.getEncryptFormat() != null && (srcPool.getPoolType() == StoragePoolType.RBD) != (destPool.getPoolType() == StoragePoolType.RBD)) { + // The KVM agent encrypts an RBD volume natively with librbd (LUKS2), and a volume on any other pool with qemu (LUKS) + volumeVo.setEncryptFormat(destPool.getPoolType() == StoragePoolType.RBD ? "luks2" : "luks"); } if (ClvmPoolManager.isClvmPoolType(destPool.getPoolType())) { clvmPoolManager.setClvmLockHostId(volume.getId(), ep.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 7eee32b9f1b5..fa92bb541c43 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -609,7 +609,7 @@ public EndPoint select(DataObject object, StorageAction action, boolean encrypti } case MIGRATEVOLUME: { VolumeInfo volume = (VolumeInfo) object; - if (volume.getHypervisorType() == Hypervisor.HypervisorType.Hyperv || volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + if (List.of(Hypervisor.HypervisorType.VMware, Hypervisor.HypervisorType.Hyperv, Hypervisor.HypervisorType.KVM).contains(volume.getHypervisorType())) { VirtualMachine vm = volume.getAttachedVM(); if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { Long hostId = vm.getHostId(); diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 3ba2e4d5789e..24a133105ae7 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -128,7 +128,14 @@ Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat Requires: python3-libnbd +Requires: python3-rados +Requires: python3-rbd +Requires: ceph-common Requires: socat +Requires: openssl +Requires: bzip2 +Requires: gzip +Requires: unzip Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec index cdfc5a72a34e..1dafc84a37f4 100644 --- a/packaging/suse15/cloud.spec +++ b/packaging/suse15/cloud.spec @@ -120,6 +120,9 @@ Requires: cifs-utils Requires: (edk2-ovmf or qemu-ovmf-x86_64) Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) +Requires: (python3-rados or python3-ceph-common) +Requires: (python3-rbd or python3-ceph-common) +Requires: ceph-common Requires: (qemu-img or qemu-tools) Requires: qemu-kvm Requires: cryptsetup @@ -127,6 +130,10 @@ Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat +Requires: openssl +Requires: bzip2 +Requires: gzip +Requires: unzip Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java index 511f0ccb7114..eac789d6d91d 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java @@ -27,9 +27,21 @@ public final class NASBackupChainKeys { /** UUID of the parent backup (full or previous incremental). Empty for full backups. */ public static final String PARENT_BACKUP_ID = "nas.parent_backup_id"; - /** QEMU dirty-bitmap name created by this backup, used as the {@code } reference for the next one. */ + /** + * Name of the point-in-time anchor this backup created, used as the parent reference for the + * next backup in the chain. Its meaning depends on the chain format: for {@link #FORMAT_QCOW2} + * chains it is the QEMU dirty-bitmap name used as the {@code } reference; for + * {@link #FORMAT_RBD_DIFF} chains it is the RBD snapshot name used as {@code --from-snap}. + */ public static final String BITMAP_NAME = "nas.bitmap_name"; + /** + * On-NAS format of the chain this backup belongs to: {@link #FORMAT_QCOW2} (rebased qcow2 + * files, the file-based storage path) or {@link #FORMAT_RBD_DIFF} ({@code rbd export-diff} + * streams, the Ceph/RBD path). Absent on legacy-full backups, which are always qcow2. + */ + public static final String CHAIN_FORMAT = "nas.chain_format"; + /** Identifier shared by every backup in the same chain (the full anchors a chain; its incrementals inherit the id). */ public static final String CHAIN_ID = "nas.chain_id"; @@ -48,6 +60,31 @@ public final class NASBackupChainKeys { // "legacy-full" mode token (which sets make_checkpoint=0). public static final String TYPE_LEGACY_FULL = "legacy-full"; + /** + * In-memory backup-method sentinels used by {@code ChainDecision.method}: how a backup is + * taken, independent of whether it is a full or an incremental ({@code ChainDecision.mode}). + * + *

{@link #METHOD_QCOW2}: libvirt push-mode backup; incrementals come from QEMU persistent + * dirty bitmaps (libvirt checkpoints), which only qcow2 volumes can carry.

+ */ + public static final String METHOD_QCOW2 = "qcow2"; + + /** + * Content-based libvirt pull-mode backup, used for raw block-device storage such as + * LINSTOR/DRBD which cannot carry persistent dirty bitmaps. A full is written as a sparse + * qcow2 in a single pass; an incremental is a qcow2 overlay on the disk's NBD export, + * safe-rebased onto the parent so it holds only the clusters that differ. + */ + public static final String METHOD_CONTENT = "content"; + + /** + * Ceph-native backup, used for RBD volumes. They are raw and cannot carry a persistent dirty + * bitmap, so each cycle anchors on an RBD snapshot and exports the delta with + * {@code rbd export-diff}. Unlike the other methods this one writes {@link #FORMAT_RBD_DIFF} + * streams rather than qcow2 files, which is what {@link #CHAIN_FORMAT} records for restore. + */ + public static final String METHOD_RBD = "rbd"; + /** * VM-scoped detail (stored in {@code vm_instance_details}) holding the QEMU dirty-bitmap * name that currently exists on the running VM and is therefore the only valid parent @@ -57,6 +94,10 @@ public final class NASBackupChainKeys { */ public static final String VM_ACTIVE_CHECKPOINT_ID = "nas.active_checkpoint_id"; + /** Chain format values persisted under {@link #CHAIN_FORMAT}; also the on-NAS file extension. */ + public static final String FORMAT_QCOW2 = "qcow2"; + public static final String FORMAT_RBD_DIFF = "rbddiff"; + private NASBackupChainKeys() { } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 8e6d33c4e668..181f74a2c57a 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -85,6 +85,13 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co private static final int CHAIN_LOCK_TIMEOUT_SECONDS = 300; + /** + * Prefix of the RBD snapshots this provider creates as chain anchors. Deliberately distinct + * from the names CloudStack volume snapshots use, so an operator (and the agent's cleanup) can + * tell a backup anchor apart from a user snapshot on the same image. + */ + private static final String RBD_SNAPSHOT_PREFIX = "csbackup-"; + ConfigKey NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class, "nas.backup.restore.mount.timeout", "30", @@ -209,13 +216,17 @@ protected Host getVMHypervisorHost(VirtualMachine vm) { /** * Returned by {@link #decideChain(VirtualMachine)} to describe the next backup's place in - * the chain: full vs incremental, the bitmap name to create, and (for incrementals) the - * parent bitmap and parent file path. + * the chain: full vs incremental, how it is taken, the bitmap name to create, and (for + * incrementals) the parent bitmap and parent file path. */ static final class ChainDecision { final String mode; // "full" or "incremental" + final String method; // NASBackupChainKeys.METHOD_QCOW2, METHOD_CONTENT or METHOD_RBD + // On-NAS format of the chain this backup belongs to: NASBackupChainKeys.FORMAT_QCOW2 for + // the libvirt-checkpoint path, FORMAT_RBD_DIFF for the Ceph-native rbd export-diff path. + final String format; final String bitmapNew; - final String bitmapParent; // null for full + final String bitmapParent; // the anchor to read changes since; null for a full // Per-volume parent backup file paths, one per current VM volume in deviceId order. // null/empty for full. Each volume needs its own parent file because backup files // are named after each volume's own UUID (root..qcow2 / datadisk..qcow2). @@ -223,9 +234,11 @@ static final class ChainDecision { final String chainId; // chain identifier this backup belongs to final int chainPosition; // 0 for full, N for the Nth incremental in the chain - private ChainDecision(String mode, String bitmapNew, String bitmapParent, List parentPaths, - String chainId, int chainPosition) { + private ChainDecision(String mode, String method, String format, String bitmapNew, String bitmapParent, + List parentPaths, String chainId, int chainPosition) { this.mode = mode; + this.method = method; + this.format = format; this.bitmapNew = bitmapNew; this.bitmapParent = bitmapParent; this.parentPaths = parentPaths; @@ -234,8 +247,29 @@ private ChainDecision(String mode, String bitmapNew, String bitmapParent, List parentPaths, String chainId, int chainPosition) { - return new ChainDecision(NASBackupChainKeys.TYPE_INCREMENTAL, bitmapNew, bitmapParent, - parentPaths, chainId, chainPosition); + return new ChainDecision(NASBackupChainKeys.TYPE_INCREMENTAL, NASBackupChainKeys.METHOD_QCOW2, + NASBackupChainKeys.FORMAT_QCOW2, bitmapNew, bitmapParent, parentPaths, chainId, chainPosition); } boolean isIncremental() { @@ -260,6 +295,31 @@ boolean isIncremental() { boolean isLegacyFull() { return NASBackupChainKeys.TYPE_LEGACY_FULL.equals(mode); } + + boolean isRbd() { + return NASBackupChainKeys.METHOD_RBD.equals(method); + } + + /** + * The same decision re-cast as the start of a brand new chain, keeping the anchor the agent + * actually created and the storage flavour. Used when the agent reports it could not + * proceed as an incremental and completed a full instead. + */ + ChainDecision asFullRestart() { + return isRbd() ? rbdFullStart(bitmapNew) : fullStart(bitmapNew); + } + } + + /** + * Which incremental mechanism, if any, a VM's volumes can support. + */ + protected enum IncrementalFlavor { + /** Every volume is on file-based qcow2 storage: libvirt checkpoints / QEMU dirty bitmaps. */ + QCOW2, + /** Every volume is on Ceph/RBD: RBD snapshots + {@code rbd export-diff}. */ + RBD, + /** Mixed or otherwise incapable storage — the VM stays on the full-only (legacy) path. */ + UNSUPPORTED } /** @@ -283,103 +343,155 @@ protected ChainDecision decideChain(VirtualMachine vm) { return ChainDecision.legacyFull(); } - // Incremental backups rely on QEMU dirty bitmaps / libvirt checkpoints, which only exist - // on file-based qcow2 storage. Storage such as Ceph-RBD and Linstor cannot carry per-disk - // checkpoints, so a VM with any volume on such a pool must stay on the full-only (legacy) - // path — otherwise an incremental attempt would fail or regress those storages. - if (!allVolumesOnCheckpointCapableStorage(vm)) { + // Pick the incremental mechanism that matches the VM's storage. Mixed or incapable + // storage stays on the full-only (legacy) path. + IncrementalFlavor flavor = getIncrementalFlavor(vm); + if (IncrementalFlavor.UNSUPPORTED.equals(flavor)) { return ChainDecision.legacyFull(); } + final boolean rbd = IncrementalFlavor.RBD.equals(flavor); - final String newBitmap = "backup-" + System.currentTimeMillis() / 1000L; + final String newAnchor = (rbd ? RBD_SNAPSHOT_PREFIX : "backup-") + System.currentTimeMillis() / 1000L; - // Stopped VMs cannot do incrementals — script will also fall back, but we make the - // decision here so we register the right type up-front. - if (VirtualMachine.State.Stopped.equals(vm.getState())) { - return ChainDecision.fullStart(newBitmap); + // A QEMU dirty bitmap needs a live QEMU process, so a stopped VM always takes a full. + // RBD snapshots are taken inside Ceph and work exactly the same whether the VM runs or + // not, so RBD chains keep their cadence across a stop/start. + if (!rbd && VirtualMachine.State.Stopped.equals(vm.getState())) { + return ChainDecision.fullStart(newAnchor); } + // The VM's active_checkpoint_id records the anchor that currently exists on the storage: + // the QEMU bitmap for qcow2 chains, the RBD snapshot for RBD chains. Null means there is + // nothing to base an incremental on (first backup, post-restore, freshly assigned offering). + String activeCheckpoint = readVmActiveCheckpoint(vm.getId()); + Integer fullEvery = NASBackupFullEvery.valueIn(vm.getDataCenterId()); if (fullEvery == null || fullEvery <= 1) { // Disabled or every-backup-is-full mode. - return ChainDecision.fullStart(newBitmap); + return fullDecision(rbd, newAnchor); } - // 1. If the VM has no active_checkpoint_id, there is no bitmap on the host to use as - // a parent. This is the case after restore (we clear it), after VM was just assigned - // to the offering, or on the very first backup. - String activeCheckpoint = readVmActiveCheckpoint(vm.getId()); + // 1. No anchor on the storage => nothing to take an incremental against. if (activeCheckpoint == null) { - return ChainDecision.fullStart(newBitmap); + return fullDecision(rbd, newAnchor); } // 2. The most-recent BackedUp backup is the only safe parent — after restore the - // next backup is always a fresh full, so anything older has a rotated-out bitmap. - // If the latest backup's bitmap doesn't match the VM's active_checkpoint_id, the + // next backup is always a fresh full, so anything older has a rotated-out anchor. + // If the latest backup's anchor doesn't match the VM's active_checkpoint_id, the // chain is broken: force a full. Backup parent = findLatestBackedUpBackup(vm.getId()); if (parent == null || !activeCheckpoint.equals(readDetail(parent, NASBackupChainKeys.BITMAP_NAME))) { LOG.debug("VM {} latest backup does not match active_checkpoint_id={} — forcing full", vm.getInstanceName(), activeCheckpoint); - return ChainDecision.fullStart(newBitmap); + return fullDecision(rbd, newAnchor); + } + + // A chain cannot change storage flavour halfway through: the on-NAS files of a qcow2 chain + // and of an RBD chain are not interchangeable. A volume migration between pool types + // therefore starts a new chain in the new format. + String parentFormat = chainFormatOf(parent); + String requiredFormat = rbd ? NASBackupChainKeys.FORMAT_RBD_DIFF : NASBackupChainKeys.FORMAT_QCOW2; + if (!requiredFormat.equals(parentFormat)) { + LOG.debug("VM {} parent backup {} is a {} chain but the VM's storage now needs {} — forcing full", + vm.getInstanceName(), parent.getUuid(), parentFormat, requiredFormat); + return fullDecision(rbd, newAnchor); } String parentChainId = readDetail(parent, NASBackupChainKeys.CHAIN_ID); int parentChainPosition = chainPosition(parent); if (parentChainId == null || parentChainPosition == Integer.MAX_VALUE) { - return ChainDecision.fullStart(newBitmap); + return fullDecision(rbd, newAnchor); } // Force a fresh full when the chain has reached the configured length. if (parentChainPosition + 1 >= fullEvery) { - return ChainDecision.fullStart(newBitmap); + return fullDecision(rbd, newAnchor); } // The script needs the parent backup's on-NAS file path PER VOLUME so it can rebase // each new qcow2 onto the matching parent. The paths are stored relative to the NAS // mount root — the script resolves them inside its mount session. When alignment // fails (volume count changed, etc.) compose returns null and we fall back to full - // so we don't risk corrupting the chain. - List parentPaths = composeParentBackupPaths(parent, vm.getId()); + // so we don't risk corrupting the chain. RBD chains don't rebase, but the same call + // still validates that the parent's volume layout still matches the VM's. + List parentPaths = composeParentBackupPaths(parent, vm.getId(), requiredFormat); if (parentPaths == null) { LOG.debug("VM {} parent backup {} volume layout no longer matches current VM — forcing full", vm.getInstanceName(), parent.getUuid()); - return ChainDecision.fullStart(newBitmap); + return fullDecision(rbd, newAnchor); } - return ChainDecision.incremental(newBitmap, activeCheckpoint, parentPaths, - parentChainId, parentChainPosition + 1); + return rbd + ? ChainDecision.rbdIncremental(newAnchor, activeCheckpoint, parentChainId, parentChainPosition + 1) + : ChainDecision.incremental(newAnchor, activeCheckpoint, parentPaths, parentChainId, parentChainPosition + 1); + } + + /** Start of a fresh chain in the format the VM's storage calls for. */ + private ChainDecision fullDecision(boolean rbd, String newAnchor) { + return rbd ? ChainDecision.rbdFullStart(newAnchor) : ChainDecision.fullStart(newAnchor); } /** - * Incremental backups require QEMU dirty bitmaps / libvirt checkpoints, which are only - * possible on file-based qcow2 storage. Returns {@code true} only when EVERY volume of the - * VM sits on HOST-scope local, {@code SharedMountPoint}, or {@code NetworkFilesystem} (NFS) - * storage. Ceph-RBD, Linstor, and any other pool that cannot carry a per-disk checkpoint - * make this return {@code false} so the caller falls back to the legacy full-only path. A - * volume whose pool can no longer be resolved is treated as incapable (safe default). + * The on-NAS format of {@code backup}'s chain. Backups taken before chain formats existed (and + * legacy fulls) carry no detail and are qcow2. */ - protected boolean allVolumesOnCheckpointCapableStorage(VirtualMachine vm) { + private String chainFormatOf(Backup backup) { + String format = readDetail(backup, NASBackupChainKeys.CHAIN_FORMAT); + return format == null ? NASBackupChainKeys.FORMAT_QCOW2 : format; + } + + /** + * Decide which incremental mechanism a VM's volumes can support. + * + *

{@link IncrementalFlavor#QCOW2} needs QEMU dirty bitmaps / libvirt checkpoints, which only + * exist on file-based qcow2 storage: HOST-scope local, {@code SharedMountPoint} or + * {@code NetworkFilesystem} (NFS) pools.

+ * + *

{@link IncrementalFlavor#RBD} uses Ceph's own primitives instead — an {@code rbd snap} per + * cycle plus {@code rbd export-diff --from-snap}, computed inside Ceph from the image's + * object-map/fast-diff. RBD images are raw and have nowhere to persist a QEMU bitmap, so they + * could never join a qcow2 chain, but they need no host-side checkpoint at all.

+ * + *

The two mechanisms produce different on-NAS files and cannot be mixed inside one backup, + * so a VM whose volumes span both (or sits on any other pool type, e.g. Linstor) is reported + * {@link IncrementalFlavor#UNSUPPORTED} and stays on the legacy full-only path. A volume whose + * pool can no longer be resolved is treated as incapable (safe default).

+ */ + protected IncrementalFlavor getIncrementalFlavor(VirtualMachine vm) { List volumes = volumeDao.findByInstance(vm.getId()); - if (volumes == null) { - return true; + if (CollectionUtils.isEmpty(volumes)) { + return IncrementalFlavor.QCOW2; } + IncrementalFlavor flavor = null; for (VolumeVO volume : volumes) { StoragePoolVO pool = primaryDataStoreDao.findById(volume.getPoolId()); if (pool == null) { LOG.debug("VM {} volume {} has no resolvable storage pool — forcing legacy full", vm.getInstanceName(), volume.getUuid()); - return false; + return IncrementalFlavor.UNSUPPORTED; } - boolean checkpointCapable = ScopeType.HOST.equals(pool.getScope()) + IncrementalFlavor volumeFlavor; + if (ScopeType.HOST.equals(pool.getScope()) || Storage.StoragePoolType.SharedMountPoint.equals(pool.getPoolType()) - || Storage.StoragePoolType.NetworkFilesystem.equals(pool.getPoolType()); - if (!checkpointCapable) { - LOG.debug("VM {} volume {} is on {} (scope {}) which cannot carry checkpoints — forcing legacy full", + || Storage.StoragePoolType.NetworkFilesystem.equals(pool.getPoolType())) { + volumeFlavor = IncrementalFlavor.QCOW2; + } else if (Storage.StoragePoolType.RBD.equals(pool.getPoolType())) { + volumeFlavor = IncrementalFlavor.RBD; + } else { + LOG.debug("VM {} volume {} is on {} (scope {}) which supports neither checkpoints nor " + + "RBD snapshots — forcing legacy full", vm.getInstanceName(), volume.getUuid(), pool.getPoolType(), pool.getScope()); - return false; + return IncrementalFlavor.UNSUPPORTED; + } + if (flavor == null) { + flavor = volumeFlavor; + } else if (!flavor.equals(volumeFlavor)) { + LOG.debug("VM {} has volumes on both checkpoint-capable and RBD storage; a single backup " + + "cannot mix the two mechanisms — forcing legacy full", vm.getInstanceName()); + return IncrementalFlavor.UNSUPPORTED; } } - return true; + return flavor; } /** @@ -420,8 +532,8 @@ private String readDetail(Backup backup, String key) { * Compose the on-NAS path of EVERY parent backup file (one per VM volume) in the same * order the script will iterate the current VM's disks (deviceId asc). Relative to the * NAS mount, matches the layout written by {@code nasbackup.sh}: - * first disk -> {@code /root..qcow2} - * others -> {@code /datadisk..qcow2} + * first disk -> {@code /root..} + * others -> {@code /datadisk..} * * Returns {@code null} if the parent's stored volume count doesn't match the current VM's * volume count. Volume attach/detach is blocked while a VM is assigned to a backup offering; @@ -430,7 +542,7 @@ private String readDetail(Backup backup, String key) { * identities — a count mismatch is the only way to reach this branch with a non-null * active_checkpoint_id. */ - private List composeParentBackupPaths(Backup parent, long vmId) { + private List composeParentBackupPaths(Backup parent, long vmId, String extension) { // backupPath is stored as externalId by createBackupObject — e.g. // "i-2-1234-VM/2026.04.27.13.45.00". String dir = parent.getExternalId(); @@ -457,7 +569,7 @@ private List composeParentBackupPaths(Backup parent, long vmId) { for (int i = 0; i < parentVols.size(); i++) { String volPath = parentVols.get(i).getPath(); String prefix = (i == 0) ? "root" : "datadisk"; - paths.add(dir + "/" + prefix + "." + volPath + ".qcow2"); + paths.add(dir + "/" + prefix + "." + volPath + "." + extension); } return paths; } @@ -477,6 +589,12 @@ private void persistChainMetadata(Backup backup, ChainDecision decision, String backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.BITMAP_NAME, bitmapFromAgent, true)); } backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.CHAIN_ID, decision.chainId, true)); + // Restore has to know how to read the files back: rebased qcow2, or an ordered replay of + // rbd export-diff streams. Only recorded for RBD chains — no detail means qcow2. + if (NASBackupChainKeys.FORMAT_RBD_DIFF.equals(decision.format)) { + backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.CHAIN_FORMAT, + NASBackupChainKeys.FORMAT_RBD_DIFF, true)); + } backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.CHAIN_POSITION, String.valueOf(decision.chainPosition), true)); // Backup full-vs-incremental type lives on backup.type (set by takeBackup) — single @@ -579,11 +697,15 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce command.setMountOptions(backupRepository.getMountOptions()); command.setQuiesce(quiesceVM); command.setMode(decision.mode); + command.setMethod(decision.method); command.setBitmapNew(decision.bitmapNew); command.setBitmapParent(decision.bitmapParent); command.setParentPaths(decision.parentPaths); - if (VirtualMachine.State.Stopped.equals(vm.getState())) { + // The agent needs the per-volume pool + path to reach the disks itself whenever it cannot + // read them out of a running libvirt domain: for a stopped VM, and for every RBD backup + // (the RBD path drives `rbd` against the images directly, running or not). + if (VirtualMachine.State.Stopped.equals(vm.getState()) || decision.isRbd()) { List vmVolumes = volumeDao.findByInstance(vm.getId()); vmVolumes.sort(Comparator.comparing(Volume::getDeviceId)); Pair, List> volumePoolsAndPaths = getVolumePoolsAndPaths(vmVolumes); @@ -614,7 +736,7 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce // backup as a full and start a new chain. ChainDecision effective = decision; if (answer.getIncrementalFallback()) { - effective = ChainDecision.fullStart(decision.bitmapNew); + effective = decision.asFullRestart(); backupVO.setType("FULL"); } List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); @@ -714,6 +836,7 @@ private Pair restoreVMBackup(VirtualMachine vm, Backup backup) restoreCommand.setRestoreVolumePools(volumePoolsAndPaths.first()); restoreCommand.setRestoreVolumePaths(volumePoolsAndPaths.second()); restoreCommand.setBackupFiles(getBackupFiles(backedVolumes)); + applyBackupFormat(restoreCommand, backup); restoreCommand.setVmExists(vm.getRemoved() == null); restoreCommand.setVmState(vm.getState()); restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value()); @@ -734,6 +857,48 @@ private Pair restoreVMBackup(VirtualMachine vm, Backup backup) return new Pair<>(answer.getResult(), answer.getDetails()); } + /** + * Tell the agent how to read {@code backup} back. qcow2 chains are self-describing — every + * incremental was rebased onto its parent, so handing the agent the leaf file is enough. RBD + * chains are not: an {@code rbd export-diff} stream carries no pointer to the diff before it, + * so the whole chain from the full anchor up to this backup travels with the command and the + * agent replays it in order. + */ + private void applyBackupFormat(RestoreBackupCommand restoreCommand, Backup backup) { + if (!NASBackupChainKeys.FORMAT_RBD_DIFF.equals(chainFormatOf(backup))) { + return; + } + restoreCommand.setBackupFormat(NASBackupChainKeys.FORMAT_RBD_DIFF); + restoreCommand.setBackupChainPaths(composeRestoreChainDirs(backup)); + } + + /** + * The backup directories to replay to reconstruct {@code backup}, ordered root-first: the + * chain's full anchor, every incremental after it, and {@code backup} itself. + * + *

Ancestors that a user asked to delete are only tombstoned (Hidden) while descendants are + * alive, precisely so their files survive on the NAS for this walk. A gap here therefore means + * the chain really was broken, and replaying what is left would silently restore a mix of two + * points in time — so we refuse instead.

+ */ + private List composeRestoreChainDirs(Backup backup) { + int targetPosition = chainPosition(backup); + List dirs = new ArrayList<>(); + for (Backup member : getChainOrderedLeafToRoot(backup)) { + if (chainPosition(member) <= targetPosition) { + dirs.add(member.getExternalId()); + } + } + // getChainOrderedLeafToRoot sorts by descending chain position; restore replays root-first. + Collections.reverse(dirs); + if (targetPosition != Integer.MAX_VALUE && dirs.size() != targetPosition + 1) { + throw new CloudRuntimeException(String.format( + "Cannot restore backup %s: its chain is incomplete (%d of the %d backups it depends on are present).", + backup.getUuid(), dirs.size(), targetPosition + 1)); + } + return dirs; + } + private List getBackupFiles(List backedVolumes) { List backupFiles = new ArrayList<>(); for (Backup.VolumeInfo backedVolume : backedVolumes) { @@ -840,6 +1005,7 @@ public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeI restoreCommand.setVmState(vmNameAndState.second()); restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value()); restoreCommand.setBackupFiles(Collections.singletonList(matchingVolume.getPath())); + applyBackupFormat(restoreCommand, backup); BackupAnswer answer; try { diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index f1d5613ab7fc..f323c859e8d5 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -450,6 +450,94 @@ public void decideChainReturnsFullWhenVmHasNoActiveCheckpoint() { Assert.assertEquals(0, decision.chainPosition); } + /** Enable the zone-level incremental master switch for a decideChain test. */ + private void enableIncrementalBackups() { + ReflectionTestUtils.setField(nasBackupProvider, "NASBackupIncrementalEnabled", + new org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Boolean.class, + "nas.backup.incremental.enabled", "true", + "test override — enabled", true, + org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone)); + } + + /** Give {@code vmId} a single volume on a Ceph-RBD pool. */ + private void givenVmVolumesOnRbd(Long vmId) { + VolumeVO rbdVol = mock(VolumeVO.class); + Mockito.when(rbdVol.getPoolId()).thenReturn(9L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(rbdVol)); + StoragePoolVO rbd = mock(StoragePoolVO.class); + Mockito.when(rbd.getPoolType()).thenReturn(Storage.StoragePoolType.RBD); + Mockito.when(storagePoolDao.findById(9L)).thenReturn(rbd); + } + + /** + * A VM entirely on Ceph-RBD no longer drops to the legacy full-only path: it anchors its chain + * on an RBD snapshot and its backups are written as rbd export-diff streams. + */ + @Test + public void decideChainStartsAnRbdChainForRbdBackedVm() { + Long vmId = 60L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.lenient().when(vm.getState()).thenReturn(VMInstanceVO.State.Running); + enableIncrementalBackups(); + givenVmVolumesOnRbd(vmId); + Mockito.when(vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID)).thenReturn(null); + + NASBackupProvider.ChainDecision decision = nasBackupProvider.decideChain(vm); + + Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, decision.mode); + Assert.assertEquals(NASBackupChainKeys.METHOD_RBD, decision.method); + Assert.assertEquals(NASBackupChainKeys.FORMAT_RBD_DIFF, decision.format); + Assert.assertTrue("RBD chains anchor on an RBD snapshot, not a QEMU bitmap", + decision.bitmapNew.startsWith("csbackup-")); + Assert.assertNull("a full reads nothing from a parent anchor", decision.bitmapParent); + Assert.assertNull("RBD backups are replayed in order, not rebased onto a parent file", decision.parentPaths); + Assert.assertEquals(0, decision.chainPosition); + } + + /** + * RBD snapshots are taken inside Ceph, so a stopped VM is no different from a running one. It + * must stay on the RBD path rather than being pushed onto a qcow2 full the way a stopped VM on + * file-based storage is (which has no live QEMU to hold a dirty bitmap). + */ + @Test + public void decideChainKeepsStoppedRbdVmOnTheRbdPath() { + Long vmId = 61L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.lenient().when(vm.getState()).thenReturn(VMInstanceVO.State.Stopped); + enableIncrementalBackups(); + givenVmVolumesOnRbd(vmId); + Mockito.when(vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID)).thenReturn(null); + + NASBackupProvider.ChainDecision decision = nasBackupProvider.decideChain(vm); + + Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, decision.mode); + Assert.assertEquals(NASBackupChainKeys.METHOD_RBD, decision.method); + Assert.assertEquals(NASBackupChainKeys.FORMAT_RBD_DIFF, decision.format); + } + + /** + * When the agent reports it completed a full after failing to proceed as an incremental, the + * backup is re-registered as the start of a new chain — in the same storage flavour, so an RBD + * backup is not accidentally recorded as a qcow2 one. + */ + @Test + public void asFullRestartKeepsTheRbdFlavour() { + NASBackupProvider.ChainDecision incremental = + NASBackupProvider.ChainDecision.rbdIncremental("csbackup-2", "csbackup-1", "chain-1", 3); + + NASBackupProvider.ChainDecision restarted = incremental.asFullRestart(); + + Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, restarted.mode); + Assert.assertEquals(NASBackupChainKeys.METHOD_RBD, restarted.method); + Assert.assertEquals(NASBackupChainKeys.FORMAT_RBD_DIFF, restarted.format); + Assert.assertEquals("csbackup-2", restarted.bitmapNew); + Assert.assertEquals(0, restarted.chainPosition); + } + // -- incremental storage-capability guard (Ceph-RBD / Linstor stay on legacy full) ---- /** @@ -457,7 +545,7 @@ public void decideChainReturnsFullWhenVmHasNoActiveCheckpoint() { * volume sits on NFS / HOST-scope local / SharedMountPoint is checkpoint-capable. */ @Test - public void allVolumesOnCheckpointCapableStorageTrueForNfsHostAndSharedMount() { + public void getIncrementalFlavorIsQcow2ForNfsHostAndSharedMount() { Long vmId = 55L; VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getId()).thenReturn(vmId); @@ -480,15 +568,15 @@ public void allVolumesOnCheckpointCapableStorageTrueForNfsHostAndSharedMount() { Mockito.when(storagePoolDao.findById(2L)).thenReturn(host); Mockito.when(storagePoolDao.findById(3L)).thenReturn(smp); - Assert.assertTrue(nasBackupProvider.allVolumesOnCheckpointCapableStorage(vm)); + Assert.assertEquals(NASBackupProvider.IncrementalFlavor.QCOW2, nasBackupProvider.getIncrementalFlavor(vm)); } /** - * A single volume on Ceph-RBD (or any pool that cannot carry a per-disk checkpoint) forces - * the whole VM onto the legacy full-only path — avoids regressing RBD/Linstor storages. + * The two mechanisms produce different on-NAS files, so a VM straddling file-based storage and + * Ceph-RBD cannot use either and stays on the legacy full-only path. */ @Test - public void allVolumesOnCheckpointCapableStorageFalseWhenAnyVolumeOnRbd() { + public void getIncrementalFlavorIsUnsupportedForMixedFileAndRbdStorage() { Long vmId = 56L; VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getId()).thenReturn(vmId); @@ -506,12 +594,54 @@ public void allVolumesOnCheckpointCapableStorageFalseWhenAnyVolumeOnRbd() { Mockito.when(storagePoolDao.findById(1L)).thenReturn(nfs); Mockito.when(storagePoolDao.findById(9L)).thenReturn(rbd); - Assert.assertFalse(nasBackupProvider.allVolumesOnCheckpointCapableStorage(vm)); + Assert.assertEquals(NASBackupProvider.IncrementalFlavor.UNSUPPORTED, nasBackupProvider.getIncrementalFlavor(vm)); + } + + /** + * Every volume on Ceph-RBD selects the Ceph-native mechanism: an RBD snapshot per cycle plus + * {@code rbd export-diff}, instead of a libvirt checkpoint that a raw RBD image cannot hold. + */ + @Test + public void getIncrementalFlavorIsRbdWhenEveryVolumeIsOnRbd() { + Long vmId = 58L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + + VolumeVO rootVol = mock(VolumeVO.class); + Mockito.when(rootVol.getPoolId()).thenReturn(9L); + VolumeVO dataVol = mock(VolumeVO.class); + Mockito.when(dataVol.getPoolId()).thenReturn(10L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(rootVol, dataVol)); + + StoragePoolVO rbd = mock(StoragePoolVO.class); + Mockito.when(rbd.getPoolType()).thenReturn(Storage.StoragePoolType.RBD); + StoragePoolVO otherRbd = mock(StoragePoolVO.class); + Mockito.when(otherRbd.getPoolType()).thenReturn(Storage.StoragePoolType.RBD); + Mockito.when(storagePoolDao.findById(9L)).thenReturn(rbd); + Mockito.when(storagePoolDao.findById(10L)).thenReturn(otherRbd); + + Assert.assertEquals(NASBackupProvider.IncrementalFlavor.RBD, nasBackupProvider.getIncrementalFlavor(vm)); + } + + /** Linstor supports neither mechanism, so it keeps the legacy full-only behaviour. */ + @Test + public void getIncrementalFlavorIsUnsupportedForLinstor() { + Long vmId = 59L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + VolumeVO vol = mock(VolumeVO.class); + Mockito.when(vol.getPoolId()).thenReturn(1L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(vol)); + StoragePoolVO linstor = mock(StoragePoolVO.class); + Mockito.when(linstor.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor); + Mockito.when(storagePoolDao.findById(1L)).thenReturn(linstor); + + Assert.assertEquals(NASBackupProvider.IncrementalFlavor.UNSUPPORTED, nasBackupProvider.getIncrementalFlavor(vm)); } /** A volume whose storage pool can no longer be resolved is treated as incapable (safe). */ @Test - public void allVolumesOnCheckpointCapableStorageFalseWhenPoolUnresolvable() { + public void getIncrementalFlavorIsUnsupportedWhenPoolUnresolvable() { Long vmId = 57L; VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getId()).thenReturn(vmId); @@ -520,7 +650,7 @@ public void allVolumesOnCheckpointCapableStorageFalseWhenPoolUnresolvable() { Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(vol)); Mockito.when(storagePoolDao.findById(1L)).thenReturn(null); - Assert.assertFalse(nasBackupProvider.allVolumesOnCheckpointCapableStorage(vm)); + Assert.assertEquals(NASBackupProvider.IncrementalFlavor.UNSUPPORTED, nasBackupProvider.getIncrementalFlavor(vm)); } // -- restore clears active_checkpoint_id --------------------------------------------- diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 0ee59c95da39..08b5851d83f6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -53,7 +53,7 @@ public Boolean hasHeartBeat() { } if (!heartBeatCheckResult) { - logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools); + logger.warn("All checks with KVMHAChecker for {} considered it as dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostAndPools); } return heartBeatCheckResult; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 9f1b849e9727..4ebc67dcda0a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -19,7 +19,9 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.ha.HighAvailabilityManager; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; import org.libvirt.Connect; import org.libvirt.LibvirtException; import org.libvirt.StoragePool; @@ -38,9 +40,15 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; + private final String hostGuid; - public KVMHAMonitor(String host) { + /** + * @param host the private IP address of the host, which names its heartbeat on NFS and SharedMountPoint pools + * @param hostGuid the GUID of the host as the management server knows it, which names its heartbeat on RBD pools + */ + public KVMHAMonitor(String host, String hostGuid) { hostPrivateIp = host; + this.hostGuid = hostGuid; rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } @@ -54,7 +62,11 @@ public void removeStoragePool(String uuid) { synchronized (haStoragePools) { HAStoragePool pool = haStoragePools.get(uuid); if (pool != null) { - Script.runSimpleBashScript("umount " + pool.getMountDestPath()); + if (StringUtils.isNotEmpty(pool.getMountDestPath())) { + Script umount = new Script("umount", logger); + umount.add(pool.getMountDestPath()); + umount.execute(); + } haStoragePools.remove(uuid); } } @@ -101,7 +113,8 @@ protected void runHeartBeat() { private String executePoolHeartBeatCommand(String uuid, HAStoragePool primaryStoragePool, String result) { for (int attempt = 1; attempt <= _heartBeatUpdateMaxTries; attempt++) { - result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); + String heartBeatHostId = StoragePoolType.RBD.equals(primaryStoragePool.getPool().getType()) ? hostGuid : hostPrivateIp; + result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, heartBeatHostId, true); if (result == null) { break; } @@ -130,7 +143,7 @@ private void checkForNotExistingLibvirtStoragePools(Set removedPools, St removedPools.add(uuid); } - logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); + logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 4281036d9456..38f293a2a9b2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -19,6 +19,7 @@ import static com.cloud.host.Host.HOST_CDROM_MAX_COUNT; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_RBD_VOLUME_ENCRYPTION; import static com.cloud.host.Host.HOST_VDDK_LIB_DIR; import static com.cloud.host.Host.HOST_VDDK_SUPPORT; import static com.cloud.host.Host.HOST_VDDK_VERSION; @@ -91,6 +92,7 @@ import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; import org.apache.cloudstack.utils.linux.CPUStat; import org.apache.cloudstack.utils.linux.KVMHostInfo; @@ -1457,7 +1459,9 @@ public boolean configure(final String name, final Map params) th final String[] info = NetUtils.getNetworkParams(privateNic); - kvmhaMonitor = new KVMHAMonitor(info[0]); + // the GUID of the host as the management server knows it, as in Agent.getResourceGuid() + final String hostGuid = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.GUID) + "-" + name; + kvmhaMonitor = new KVMHAMonitor(info[0], hostGuid); final Thread haMonitorThread = new Thread(kvmhaMonitor); haMonitorThread.start(); @@ -3882,7 +3886,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { if (volumeObjectTO.requiresEncryption() && pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) { String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase()); - DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat())); + // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. + String encryptEngine = (pool.getType() == StoragePoolType.RBD) ? "librbd" : null; + DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()), encryptEngine); disk.setLibvirtDiskEncryptDetails(encryptDetails); } } @@ -4410,6 +4416,7 @@ public StartupCommand[] initialize() { cmd.setGatewayIpAddress(localGateway); cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); + cmd.getHostDetails().put(HOST_RBD_VOLUME_ENCRYPTION, String.valueOf(hostSupportsRbdVolumeEncryption())); cmd.setHostTags(getHostTags()); boolean instanceConversionSupported = hostSupportsInstanceConversion(); cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); @@ -6195,7 +6202,10 @@ public boolean isHostSecured() { } /** - * Test host for volume encryption support + * Test host for qemu-native LUKS volume encryption (qemu-img LUKS support + cryptsetup), + * reported as {@code host.volume.encryption}. RBD/librbd encryption support is a separate + * capability, reported as {@code host.volume.encryption.rbd} + * (see {@link #hostSupportsRbdVolumeEncryption()}). * @return boolean */ public boolean hostSupportsVolumeEncryption() { @@ -6220,6 +6230,13 @@ public boolean hostSupportsVolumeEncryption() { return true; } + /** + * Test host for librbd native LUKS encryption support (rbd CLI with the encryption subcommand). + */ + public boolean hostSupportsRbdVolumeEncryption() { + return new RbdEncryption().isSupported(); + } + public boolean isSecureMode(String bootMode) { if (StringUtils.isNotBlank(bootMode) && "secure".equalsIgnoreCase(bootMode)) { return true; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStorageVolumeXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStorageVolumeXMLParser.java index 00126a07cd3a..6be955a5931e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStorageVolumeXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStorageVolumeXMLParser.java @@ -48,7 +48,7 @@ public LibvirtStorageVolumeDef parseStorageVolumeXML(String volXML) { String VolName = getTagValue("name", rootElement); Element target = (Element)rootElement.getElementsByTagName("target").item(0); - String format = getAttrValue("type", "format", target); + String format = getAttrValue("format", "type", target); Long capacity = Long.parseLong(getTagValue("capacity", rootElement)); return new LibvirtStorageVolumeDef(VolName, capacity, LibvirtStorageVolumeDef.VolumeFormat.getFormat(format), null, null); } catch (ParserConfigurationException e) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 74529d9d5fa2..439e4f663416 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -788,14 +788,21 @@ public static class DiskDef { public static class LibvirtDiskEncryptDetails { String passphraseUuid; QemuObject.EncryptFormat encryptFormat; + String engine; // optional libvirt encryption engine (e.g. "librbd"); null => libvirt/qemu default public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat) { + this(passphraseUuid, encryptFormat, null); + } + + public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat, String engine) { this.passphraseUuid = passphraseUuid; this.encryptFormat = encryptFormat; + this.engine = engine; } public String getPassphraseUuid() { return this.passphraseUuid; } public QemuObject.EncryptFormat getEncryptFormat() { return this.encryptFormat; } + public String getEngine() { return this.engine; } } public static class DiskGeometry { @@ -1446,7 +1453,11 @@ public String toString() { } if (encryptDetails != null) { - diskBuilder.append("\n"); + diskBuilder.append("\n"); diskBuilder.append("\n"); diskBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java index 964e6591878c..9d47f497196e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java @@ -47,6 +47,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.RbdUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @@ -98,7 +99,7 @@ public Answer execute(final BackupSnapshotCommand command, final LibvirtComputin if (primaryPool.getType() == StoragePoolType.RBD) { try { final Rados r = new Rados(primaryPool.getAuthUserName()); - r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("mon_host", RbdUtils.getRbdMonitors(primaryPool.getSourceHost(), primaryPool.getSourcePort())); r.confSet("key", primaryPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); r.connect(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index b132f6d98ce9..745686ce33de 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -22,6 +22,8 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; @@ -30,6 +32,8 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.script.Script; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -50,7 +54,11 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L if (primaryPool.isPoolSupportHA()) { final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + final String vmActivityCheckPath = getVmActivityCheckPath(pool.getType(), libvirtComputingResource); + if (vmActivityCheckPath == null) { + return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: unable to find the script which checks the VM activity on a storage pool of type " + pool.getType()); + } + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); @@ -67,4 +75,16 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L } return new Answer(command, false, "Unsupported Storage"); } + + /** + * The script which checks the VM activity depends on the type of the storage pool: a RBD pool has no shared mount point to look at, + * so its script checks the watchers of the volumes, instead of the files of the volumes as the script of the other pools. + */ + private String getVmActivityCheckPath(final StoragePoolType poolType, final LibvirtComputingResource libvirtComputingResource) { + if (StoragePoolType.RBD.equals(poolType)) { + final String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + return Script.findScript(kvmScriptsDir, "kvmvmactivity_rbd.py"); + } + return libvirtComputingResource.getVmActivityCheckPath(); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 6788516df741..3a42f230f845 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -50,7 +50,8 @@ public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.SharedMountPoint); + Storage.StoragePoolType.SharedMountPoint, + Storage.StoragePoolType.RBD); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { @@ -64,14 +65,25 @@ public Answer execute(final CheckVolumeCommand command, final LibvirtComputingRe if (STORAGE_POOL_TYPES_SUPPORTED.contains(storageFilerTO.getType())) { final KVMPhysicalDisk vol = pool.getPhysicalDisk(srcFile); final String path = vol.getPath(); - try { - KVMPhysicalDisk.checkQcow2File(path); - } catch (final CloudRuntimeException e) { - return new CheckVolumeAnswer(command, false, "", 0, getVolumeDetails(pool, vol)); + final boolean isRbd = Storage.StoragePoolType.RBD.equals(storageFilerTO.getType()); + + Map volumeDetails = getVolumeDetails(pool, vol); + if (MapUtils.isEmpty(volumeDetails)) { + return new Answer(command, false, "Unable to read the volume on the storage pool"); + } + + if (!isRbd) { + try { + KVMPhysicalDisk.checkQcow2File(path); + } catch (final CloudRuntimeException e) { + return new CheckVolumeAnswer(command, false, "", 0, volumeDetails); + } } - long size = KVMPhysicalDisk.getVirtualSizeFromFile(path); - return new CheckVolumeAnswer(command, true, "", size, getVolumeDetails(pool, vol)); + // Images on RBD are raw and the path is an image name that qemu-img cannot open + // without the rbd: URI, so take the size libvirt already reported for the volume. + long size = isRbd ? vol.getVirtualSize() : KVMPhysicalDisk.getVirtualSizeFromFile(path); + return new CheckVolumeAnswer(command, true, "", size, volumeDetails); } else { return new Answer(command, false, "Unsupported Storage Pool"); } @@ -122,6 +134,9 @@ private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk try { QemuImg qemu = new QemuImg(0); QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + if (Storage.StoragePoolType.RBD.equals(pool.getType())) { + qemuFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath()), disk.getFormat()); + } return qemu.info(qemuFile, secure); } catch (QemuImgException | LibvirtException ex) { logger.error("Failed to get info of disk file: " + ex.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java index ec900e9981e0..792737db1cf7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java @@ -43,6 +43,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.RbdUtils; import com.cloud.utils.script.Script; @ResourceWrapper(handles = ManageSnapshotCommand.class) @@ -115,7 +116,7 @@ public Answer execute(final ManageSnapshotCommand command, final LibvirtComputin if (primaryPool.getType() == StoragePoolType.RBD) { try { final Rados r = new Rados(primaryPool.getAuthUserName()); - r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet("mon_host", RbdUtils.getRbdMonitors(primaryPool.getSourceHost(), primaryPool.getSourcePort())); r.confSet("key", primaryPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); r.connect(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 85574c447417..2d56b72effda 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -26,9 +26,11 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.disconnecthook.VolumeMigrationCancelHook; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage; @@ -38,12 +40,14 @@ import java.io.IOException; import java.io.StringWriter; import java.util.Map; +import java.util.Objects; import java.util.UUID; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; @@ -56,6 +60,8 @@ import org.libvirt.LibvirtException; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; @@ -77,9 +83,18 @@ public Answer execute(final MigrateVolumeCommand command, final LibvirtComputing VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + PrimaryDataStoreTO destPrimaryDataStore = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + MigrateVolumeAnswer answer; - if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { + if (Objects.equals(Storage.StoragePoolType.PowerFlex, srcPrimaryDataStore.getPoolType())) { answer = migratePowerFlexVolume(command, libvirtComputingResource); + } else if (Objects.equals(Storage.StoragePoolType.RBD, srcPrimaryDataStore.getPoolType()) + || Objects.equals(Storage.StoragePoolType.RBD, destPrimaryDataStore.getPoolType()) + || (isFileBasedPool(srcPrimaryDataStore) && isFileBasedPool(destPrimaryDataStore))) { + // RBD, NFS, SharedMountPoint and local volumes are live migrated in place with a libvirt block copy, as both pools are + // reachable from the host the Instance is running on (the command is sent to that host). + answer = migrateRBDVolume(command, libvirtComputingResource); } else { answer = migrateRegularVolume(command, libvirtComputingResource); } @@ -87,6 +102,12 @@ public Answer execute(final MigrateVolumeCommand command, final LibvirtComputing return answer; } + private static boolean isFileBasedPool(PrimaryDataStoreTO dataStore) { + return Objects.equals(Storage.StoragePoolType.NetworkFilesystem, dataStore.getPoolType()) + || Objects.equals(Storage.StoragePoolType.SharedMountPoint, dataStore.getPoolType()) + || Objects.equals(Storage.StoragePoolType.Filesystem, dataStore.getPoolType()); + } + protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { // Source Details @@ -134,7 +155,7 @@ protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand destSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); } - String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath, destSecretUUID); + String diskdef = generateDestinationDiskXMLForPowerFlex(dm, srcVolumeId, diskFilePath, destSecretUUID); destDiskLabel = generateDestinationDiskLabel(diskdef); TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); @@ -249,7 +270,7 @@ private String generateDestinationDiskLabel(String diskXml) throws ParserConfigu return diskLabel; } - protected String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { + protected String generateDestinationDiskXMLForPowerFlex(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { final String domXml = dm.getXMLDesc(0); DocumentBuilderFactory dbFactory = ParserUtils.getSaferDocumentBuilderFactory(); @@ -374,4 +395,391 @@ protected MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand co return new MigrateVolumeAnswer(command, true, null, destPath); } + + + protected MigrateVolumeAnswer migrateRBDVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + final String vmName = command.getAttachedVmName(); + + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + PrimaryDataStoreTO sourcePool = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + PrimaryDataStoreTO destPool = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + + String destDiskLabel = null; + + VolumeMigrationCancelHook cancelHook = null; + + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + + Domain dm = null; + MigrateVolumeAnswer answer = null; + boolean destVolumeCreated = false; + KVMStoragePool destKvmPool = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = libvirtUtilitiesHelper.getConnection(); + dm = libvirtComputingResource.getDomain(conn, vmName); + if (dm == null) { + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to can not find vm: " + vmName, null); + } + + DomainInfo.DomainState domainState = dm.getInfo().state ; + if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); + } + + KVMStoragePool pool = storagePoolMgr.getStoragePool(destPool.getPoolType(), destPool.getUuid()); + destKvmPool = pool; + pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); + + KVMStoragePool srcKvmPool = storagePoolMgr.getStoragePool(sourcePool.getPoolType(), sourcePool.getUuid()); + final String sourceFilePath = generateVolumePathOnHost(srcKvmPool, sourcePool, srcVolumeObjectTO); + final String destFilePath = generateVolumePathOnHost(pool, destPool, destVolumeObjectTO); + + String srcSecretUUID = null; + String destSecretUUID = null; + if (ArrayUtils.isNotEmpty(destVolumeObjectTO.getPassphrase())) { + srcSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase()); + destSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); + if (Objects.equals(srcSecretUUID, destSecretUUID)) { + // the secret is named after the volume path and is shared by both disks: keep it for the Instance + srcSecretUUID = null; + } + } + + // A block copy between two RBD pools only has to copy the active layer, as the destination image is + // already provisioned. When NFS is involved the backing chain of the source differs from what the + // destination can reference, so the whole chain is flattened into the destination instead. + boolean srcAndDestOnRbd = Objects.equals(Storage.StoragePoolType.RBD, sourcePool.getPoolType()) + && Objects.equals(Storage.StoragePoolType.RBD, destPool.getPoolType()); + + String diskdef = generateDestinationDiskXMLForRBD(dm, sourceFilePath, destFilePath, destSecretUUID, pool, destPool.getPoolType(), + srcAndDestOnRbd); + if (diskdef == null) { + return new MigrateVolumeAnswer(command, false, + String.format("Migrate volume failed as the disk with path %s was not found in the XML domain of vm %s", sourceFilePath, vmName), null); + } + destDiskLabel = generateDestinationDiskLabel(diskdef); + + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); + TypedParameter[] parameters = new TypedParameter[1]; + parameters[0] = parameter; + + int blockCopyFlags = srcAndDestOnRbd ? Domain.BlockCopyFlags.SHALLOW : 0; + if (ArrayUtils.isNotEmpty(destVolumeObjectTO.getPassphrase())) { + // libvirt can not create (and format) an encrypted mirror itself, so the destination volume is + // created with its encryption header up front and the block copy writes into it. + KVMPhysicalDisk srcDisk = srcKvmPool.getPhysicalDisk(srcVolumeObjectTO.getPath()); + long guestSize = srcDisk.getVirtualSize(); + if (Objects.equals(Storage.StoragePoolType.RBD, sourcePool.getPoolType())) { + guestSize -= LibvirtStorageAdaptor.LUKS2_HEADER_RESERVE_BYTES; + } + createEncryptedDestinationVolume(pool, destPool, destVolumeObjectTO, guestSize); + destVolumeCreated = true; + blockCopyFlags |= Domain.BlockCopyFlags.REUSE_EXT; + } + + cancelHook = new VolumeMigrationCancelHook(dm, destDiskLabel); + libvirtComputingResource.addDisconnectHook(cancelHook); + + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.PROCESSING_IN_BACKEND); + + dm.blockCopy(destDiskLabel, diskdef, parameters, blockCopyFlags); + logger.info(String.format("Block copy has started for the volume %s : %s to %s", destDiskLabel, sourceFilePath, destFilePath)); + + answer = checkBlockJobStatus(command, dm, destDiskLabel, sourceFilePath, destFilePath, libvirtComputingResource, conn, srcSecretUUID); + if (answer != null) { + if (answer.getResult()) { + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.COMPLETED); + } else { + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.FAILED); + } + } + if (answer != null && answer.getResult()) { + answer = new MigrateVolumeAnswer(command, true, null, destVolumeObjectTO.getPath()); + + // Libvirt only detects the volume's real format (e.g. qcow2) if the destination pool + // already knew about it as such. Since the block copy wrote the file directly (not + // through libvirt's own volume-creation calls), any pool other than RBD needs an + // explicit refresh here; otherwise libvirt keeps whatever it inferred when the disk + // was first attached to this domain (or falls back to a filename-suffix guess), which + // for a volume carried over from RBD would wrongly be treated as a raw block device the + // next time the Instance is (re)started. + if (!Objects.equals(Storage.StoragePoolType.RBD, destPool.getPoolType())) { + pool.refresh(); + } + } + return answer; + } catch (Exception e) { + String msg = "Migrate volume failed due to " + e.toString(); + logger.warn(msg, e); + if (destDiskLabel != null) { + try { + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + logger.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + } + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.FAILED); + answer = new MigrateVolumeAnswer(command, false, msg, null); + return answer; + } finally { + if (cancelHook != null) { + libvirtComputingResource.removeDisconnectHook(cancelHook); + } + if (destVolumeCreated && (answer == null || !answer.getResult())) { + deleteDestinationVolume(destKvmPool, destPool, destVolumeObjectTO.getPath()); + } + if (answer != null && answer.getResult()) { + // Remove old volume after successful migration + KVMStoragePool libvirtSourcePool = storagePoolMgr.getStoragePool(sourcePool.getPoolType(), sourcePool.getUuid()); + Storage.ImageFormat srcFormat = Objects.equals(Storage.StoragePoolType.RBD, sourcePool.getPoolType()) + ? Storage.ImageFormat.RAW : Storage.ImageFormat.QCOW2; + libvirtSourcePool.deletePhysicalDisk(srcVolumeObjectTO.getPath(), srcFormat); + } + if (dm != null) { + try { + dm.free(); + } catch (LibvirtException l) { + logger.trace("Ignoring libvirt error.", l); + } + } + } + } + + + /** + * Creates the (empty) destination volume of a live migration of an encrypted volume, with its encryption + * header, so that the block copy can write into it. An RBD volume gets a native librbd LUKS2 header, a volume + * on a file based pool is a qemu LUKS encrypted QCOW2 file. + * + * @param guestSize size of the disk as seen by the Instance + */ + protected void createEncryptedDestinationVolume(KVMStoragePool pool, PrimaryDataStoreTO destPool, VolumeObjectTO destVolumeObjectTO, long guestSize) { + String path = destVolumeObjectTO.getPath(); + // a previous failed attempt may have left the volume behind + deleteDestinationVolume(pool, destPool, path); + if (Objects.equals(Storage.StoragePoolType.RBD, destPool.getPoolType())) { + // the LUKS2 header is carved out of the image, so it has to be added to what the Instance can use + pool.createPhysicalDisk(path, PhysicalDiskFormat.RAW, Storage.ProvisioningType.THIN, + guestSize + LibvirtStorageAdaptor.LUKS2_HEADER_RESERVE_BYTES, destVolumeObjectTO.getPassphrase()); + } else { + pool.createPhysicalDisk(path, PhysicalDiskFormat.QCOW2, Storage.ProvisioningType.THIN, guestSize, destVolumeObjectTO.getPassphrase()); + } + } + + protected void deleteDestinationVolume(KVMStoragePool pool, PrimaryDataStoreTO destPool, String path) { + if (pool == null) { + return; + } + try { + pool.deletePhysicalDisk(path, Objects.equals(Storage.StoragePoolType.RBD, destPool.getPoolType()) + ? Storage.ImageFormat.RAW : Storage.ImageFormat.QCOW2); + } catch (Exception e) { + logger.debug("Could not remove the destination volume {} (it may not exist): {}", path, e.getMessage()); + } + } + + /** + * Returns the path of the volume as it is referenced in the XML domain of the Instance on this host. RBD volumes + * are addressed as "/", while file based volumes (e.g. NFS) are addressed by their path on the + * local mount point of the pool. + */ + protected String generateVolumePathOnHost(KVMStoragePool kvmStoragePool, PrimaryDataStoreTO dataStore, VolumeObjectTO volumeObjectTO) { + if (Objects.equals(Storage.StoragePoolType.RBD, dataStore.getPoolType())) { + return String.format("%s/%s", dataStore.getPath(), volumeObjectTO.getPath()); + } + return String.format("%s/%s", kvmStoragePool.getLocalPath(), volumeObjectTO.getPath()); + } + + /** + * Generates the XML of the destination disk used by the libvirt block copy. The source disk is located in the XML + * domain of the Instance by its current path, and is then rewritten to point at the destination pool. Both an RBD + * (network) and an NFS (file) destination are supported, independently of the type of the source disk. + */ + /** + * Build the mirror disk XML handed to {@code blockCopy} from the disk the volume currently uses. + * + * @param shallowBlockCopy whether the copy will run with {@code VIR_DOMAIN_BLOCK_COPY_SHALLOW}, which decides + * whether the source's backing chain may be carried over to the mirror. + */ + protected String generateDestinationDiskXMLForRBD(Domain dm, String srcVolumePath, String diskFilePath, String destSecretUUID, KVMStoragePool pool, + Storage.StoragePoolType destPoolType, boolean shallowBlockCopy) + throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { + final String domXml = dm.getXMLDesc(0); + + DocumentBuilderFactory dbFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(domXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + NodeList disks = doc.getElementsByTagName("disk"); + + for (int i = 0; i < disks.getLength(); i++) { + Node diskNode = disks.item(i); + if (!isDiskNodeOfVolume(diskNode, srcVolumePath)) { + continue; + } + + logger.debug(String.format("Found disk node with source volume path %s, generating destination disk XML for %s on %s", srcVolumePath, diskFilePath, + destPoolType)); + updateDiskNodeForDestinationPool(doc, diskNode, diskFilePath, destSecretUUID, pool, destPoolType, shallowBlockCopy); + + StringWriter diskSection = new StringWriter(); + Transformer xformer = TransformerFactory.newInstance().newTransformer(); + xformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + xformer.transform(new DOMSource(diskNode), new StreamResult(diskSection)); + + return diskSection.toString(); + } + return null; + } + + /** + * True if the disk node is the one backing the volume with the given path, be it an RBD (network) or a file disk. + */ + protected boolean isDiskNodeOfVolume(Node diskNode, String srcVolumePath) { + Node diskTypeAttribute = diskNode.getAttributes().getNamedItem("type"); + if (diskTypeAttribute == null) { + return false; + } + String diskType = diskTypeAttribute.getNodeValue(); + boolean networkDisk = LibvirtVMDef.DiskDef.DiskType.NETWORK.toString().equalsIgnoreCase(diskType); + boolean fileDisk = LibvirtVMDef.DiskDef.DiskType.FILE.toString().equalsIgnoreCase(diskType); + if (!networkDisk && !fileDisk) { + return false; + } + + Node sourceNode = getChildNode(diskNode, "source"); + if (sourceNode == null) { + return false; + } + NamedNodeMap sourceAttributes = sourceNode.getAttributes(); + + if (networkDisk) { + Node protocolAttribute = sourceAttributes.getNamedItem("protocol"); + if (protocolAttribute == null || !protocolAttribute.getNodeValue().equalsIgnoreCase(LibvirtVMDef.DiskDef.DiskProtocol.RBD.toString())) { + logger.debug("Skipped disk node with protocol: " + protocolAttribute); + return false; + } + Node nameAttribute = sourceAttributes.getNamedItem("name"); + return nameAttribute != null && nameAttribute.getNodeValue().contains(srcVolumePath); + } + + Node fileAttribute = sourceAttributes.getNamedItem("file"); + return fileAttribute != null && fileAttribute.getNodeValue().contains(srcVolumePath); + } + + /** + * Rewrites the disk node so that it refers to the volume on the destination pool. The disk type, the source, the + * driver type and the Ceph authentication are all adjusted to what the destination pool requires. + */ + protected void updateDiskNodeForDestinationPool(Document doc, Node diskNode, String diskFilePath, String destSecretUUID, KVMStoragePool pool, + Storage.StoragePoolType destPoolType, boolean shallowBlockCopy) { + boolean destinationOnRbd = Objects.equals(Storage.StoragePoolType.RBD, destPoolType); + + Element diskElement = (Element)diskNode; + diskElement.setAttribute("type", destinationOnRbd ? LibvirtVMDef.DiskDef.DiskType.NETWORK.toString().toLowerCase() + : LibvirtVMDef.DiskDef.DiskType.FILE.toString().toLowerCase()); + + Node sourceNode = getChildNode(diskNode, "source"); + if (sourceNode != null) { + diskNode.removeChild(sourceNode); + } + diskNode.appendChild(destinationOnRbd ? createRbdSourceNode(doc, diskFilePath, pool) : createFileSourceNode(doc, diskFilePath)); + + // A disk cloned from a template keeps that template as its backing file, and libvirt reports the whole + // chain in the live domain XML. Only a shallow copy may carry that chain into the mirror: a full copy + // flattens every layer into the destination, so a there describes a chain the destination + // does not have. libvirt refuses that combination outright with "backingStore of mirror without + // VIR_DOMAIN_BLOCK_COPY_SHALLOW doesn't make sense", which is why the element is dropped here. + if (!shallowBlockCopy) { + Node backingStoreNode = getChildNode(diskNode, "backingStore"); + if (backingStoreNode != null) { + diskNode.removeChild(backingStoreNode); + } + } + + Node driverNode = getChildNode(diskNode, "driver"); + if (driverNode instanceof Element) { + // RBD volumes are raw, whereas volumes on a file based pool are qcow2 + ((Element)driverNode).setAttribute("type", destinationOnRbd ? "raw" : "qcow2"); + } + + // The Ceph authentication of the source pool does not apply to the destination pool, so it is always replaced + Node authNode = getChildNode(diskNode, "auth"); + if (authNode != null) { + diskNode.removeChild(authNode); + } + if (destinationOnRbd && StringUtils.isNotBlank(pool.getAuthUserName())) { + diskNode.appendChild(createAuthNode(doc, pool)); + } + + if (StringUtils.isNotBlank(destSecretUUID)) { + Node encryptionNode = getChildNode(diskNode, "encryption"); + if (encryptionNode instanceof Element) { + // librbd encrypts an RBD image natively (LUKS2), a file is encrypted by qemu (LUKS) + Element encryptionElement = (Element)encryptionNode; + if (destinationOnRbd) { + encryptionElement.setAttribute("format", "luks2"); + encryptionElement.setAttribute("engine", "librbd"); + } else { + encryptionElement.setAttribute("format", "luks"); + encryptionElement.removeAttribute("engine"); + } + } + if (encryptionNode != null) { + Node secretNode = getChildNode(encryptionNode, "secret"); + if (secretNode != null) { + Node uuidAttribute = secretNode.getAttributes().getNamedItem("uuid"); + if (uuidAttribute != null) { + uuidAttribute.setTextContent(destSecretUUID); + } + } + } + } + } + + protected Element createRbdSourceNode(Document doc, String diskFilePath, KVMStoragePool pool) { + Element sourceNode = doc.createElement("source"); + sourceNode.setAttribute("protocol", LibvirtVMDef.DiskDef.DiskProtocol.RBD.toString().toLowerCase()); + sourceNode.setAttribute("name", diskFilePath); + for (String sourceHost : pool.getSourceHost().split(",")) { + Element hostNode = doc.createElement("host"); + hostNode.setAttribute("name", sourceHost); + if (pool.getSourcePort() != 0) { + hostNode.setAttribute("port", String.valueOf(pool.getSourcePort())); + } + sourceNode.appendChild(hostNode); + } + return sourceNode; + } + + protected Element createFileSourceNode(Document doc, String diskFilePath) { + Element sourceNode = doc.createElement("source"); + sourceNode.setAttribute("file", diskFilePath); + return sourceNode; + } + + protected Element createAuthNode(Document doc, KVMStoragePool pool) { + Element authNode = doc.createElement("auth"); + authNode.setAttribute("username", pool.getAuthUserName()); + Element secretNode = doc.createElement("secret"); + secretNode.setAttribute("type", "ceph"); + secretNode.setAttribute("uuid", pool.getAuthSecretUUID()); + authNode.appendChild(secretNode); + return authNode; + } + + protected Node getChildNode(Node parent, String nodeName) { + NodeList childNodes = parent.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node childNode = childNodes.item(i); + if (nodeName.equals(childNode.getNodeName())) { + return childNode; + } + } + return null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index a43b584dd6d6..20e3891478b4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -93,6 +94,13 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR final String path = vol.getPath(); String type = notifyOnlyType; + // Encrypted RBD volumes are encrypted natively by librbd; they must be resized via + // `rbd resize --encryption-passphrase-file` so librbd grows the encrypted payload and keeps + // the LUKS header consistent. The libvirt/qemu-img resize paths below are for qemu-native + // encryption and would not handle the librbd LUKS2 layout. + final boolean rbdEncrypted = pool.getType() == StoragePoolType.RBD + && command.getPassphrase() != null && command.getPassphrase().length > 0; + if (spool.getType().equals(StoragePoolType.PowerFlex) && vol.getFormat().equals(PhysicalDiskFormat.QCOW2)) { // PowerFlex QCOW2 sizing needs to consider overhead. newSize = ScaleIOStorageAdaptor.getUsableBytesFromRawBytes(newSize); @@ -115,7 +123,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR /* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via qemu-img */ if (pool.getType() != StoragePoolType.CLVM && pool.getType() != StoragePoolType.CLVM_NG && pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex - && vol.getFormat() != PhysicalDiskFormat.QCOW2) { + && vol.getFormat() != PhysicalDiskFormat.QCOW2 && !rbdEncrypted) { logger.debug("Volume " + path + " can be resized by libvirt. Asking libvirt to resize the volume."); try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); @@ -139,11 +147,15 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR boolean vmIsRunning = isVmRunning(vmInstanceName, libvirtComputingResource); - /* when VM is offline, we use qemu-img directly to resize encrypted volumes. - If VM is online, the existing resize script will call virsh blockresize which works - with both encrypted and non-encrypted volumes. + /* when VM is offline, we use qemu-img (or rbd, for librbd-encrypted RBD) directly to resize + encrypted volumes. If VM is online, the existing resize script calls virsh blockresize, + which for an librbd-encrypted RBD disk lets qemu/librbd grow the encrypted payload and + notify the guest in one step (no passphrase needed, qemu already has the secret loaded). */ - if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { + if (rbdEncrypted && !vmIsRunning) { + logger.debug("Invoking rbd to resize an offline, encrypted (librbd) RBD volume"); + resizeRbdEncryptedVolume(pool, vol, newSize, shrinkOk, command.getPassphrase()); + } else if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { logger.debug("Invoking qemu-img to resize an offline, encrypted volume"); QemuObject.EncryptFormat encryptFormat = QemuObject.EncryptFormat.enumValue(command.getEncryptFormat()); resizeEncryptedQcowFile(vol, encryptFormat,newSize, command.getPassphrase(), libvirtComputingResource); @@ -213,6 +225,16 @@ private void resizeEncryptedQcowFile(final KVMPhysicalDisk vol, final QemuObject } } + private void resizeRbdEncryptedVolume(final KVMStoragePool pool, final KVMPhysicalDisk vol, long newSize, + boolean shrinkOk, byte[] passphrase) throws CloudRuntimeException { + try { + new RbdEncryption().resize(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getAuthSecret(), pool.getSourceDir(), vol.getName(), newSize, shrinkOk, passphrase); + } finally { + Arrays.fill(passphrase, (byte) 0); + } + } + private Answer handleMultipathSCSIResize(ResizeVolumeCommand command, KVMStoragePool pool) { ((MultipathSCSIPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize()); return new ResizeVolumeAnswer(command, true, ""); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 3839d7f6bdbd..a5456f82df83 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -20,14 +20,21 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import java.util.Locale; +import java.util.Map; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.cloudstack.backup.BackupAnswer; import org.apache.cloudstack.backup.RestoreBackupCommand; @@ -35,6 +42,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; @@ -48,6 +56,7 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage; import com.cloud.utils.Pair; +import com.cloud.utils.RbdUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import com.cloud.vm.VirtualMachine; @@ -55,12 +64,67 @@ @ResourceWrapper(handles = RestoreBackupCommand.class) public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper { private static final String BACKUP_TEMP_FILE_PREFIX = "csbackup"; + + /** Prefix of the chain anchors the RBD backup path takes; see NASBackupProvider. */ + private static final String RBD_BACKUP_SNAPSHOT_PREFIX = "csbackup-"; private static final String FILE_PATH_PLACEHOLDER = "%s/%s"; // Detects whether a qcow2 file references a parent in its backing-file metadata. // Returns 0 (true) when a backing file is present, 1 when not. Uses --output=json // so the test is robust to qemu-img version differences in human-readable output. private static final String QEMU_IMG_HAS_BACKING_COMMAND = "qemu-img info --output=json %s 2>/dev/null | grep -q '\"backing-filename\"'"; + private static final String FORMAT_RBD_DIFF = "rbddiff"; + private static final String FORMAT_QCOW2 = "qcow2"; + // `rbd info --format json` reports the image size in bytes. + private static final Pattern RBD_INFO_SIZE = Pattern.compile("\"size\"\\s*:\\s*(\\d+)"); + private static final Pattern RBD_SNAP_NAME = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\""); + + // Matches the features CloudStack itself creates RBD volumes with + // (LibvirtStorageAdaptor.RBD_FEATURES), so a restored image behaves like any other. object-map + // and fast-diff in particular are what make a later `rbd export-diff` cheap. + private static final List RBD_IMAGE_FEATURES = + List.of("layering", "exclusive-lock", "object-map", "fast-diff", "deep-flatten"); + + /** + * How the backup being restored is laid out on the NAS. + * + *

A qcow2 backup is a single self-describing file per volume: incrementals were rebased onto + * their parent, so following the backing chain from the leaf is enough. An {@code rbddiff} + * backup is a chain of {@code rbd export-diff} streams with no link between them, so the whole + * chain travels with the command and every diff is replayed in order.

+ * + *

Held as a value object rather than wrapper state: command wrappers are shared singletons + * and must stay stateless.

+ */ + private static final class BackupLayout { + private final String format; + private final List chainDirs; // root-first; exactly one entry for qcow2 + + private BackupLayout(RestoreBackupCommand command) { + this.format = FORMAT_RBD_DIFF.equals(command.getBackupFormat()) ? FORMAT_RBD_DIFF : FORMAT_QCOW2; + List dirs = command.getBackupChainPaths(); + this.chainDirs = (dirs == null || dirs.isEmpty()) + ? Collections.singletonList(command.getBackupPath()) + : dirs; + } + + private boolean isRbdDiff() { + return FORMAT_RBD_DIFF.equals(format); + } + + /** + * Every on-NAS file that has to be read for one volume, oldest first. For qcow2 that is the + * single leaf file; for rbddiff, the full anchor followed by each incremental. + */ + private List filesFor(String mountDirectory, String backupFile, String diskType) { + String fileName = String.format("%s.%s.%s", diskType.toLowerCase(Locale.ROOT), backupFile, format); + List files = new ArrayList<>(chainDirs.size()); + for (String dir : chainDirs) { + files.add(String.format(FILE_PATH_PLACEHOLDER, String.format(FILE_PATH_PLACEHOLDER, mountDirectory, dir), fileName)); + } + return files; + } + } private String getVolumeUuidFromPath(String volumePath, PrimaryDataStoreTO volumePool) { if (Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) { @@ -79,7 +143,6 @@ private String getVolumeUuidFromPath(String volumePath, PrimaryDataStoreTO volum @Override public Answer execute(RestoreBackupCommand command, LibvirtComputingResource serverResource) { String vmName = command.getVmName(); - String backupPath = command.getBackupPath(); String backupRepoAddress = command.getBackupRepoAddress(); String backupRepoType = command.getBackupRepoType(); String mountOptions = command.getMountOptions(); @@ -92,6 +155,7 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser int timeout = command.getWait() > 0 ? command.getWait() * 1000 : serverResource.getCmdsTimeout(); KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); List backupFiles = command.getBackupFiles(); + BackupLayout layout = new BackupLayout(command); String newVolumeId = null; try { @@ -102,12 +166,12 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser String backupFile = backupFiles.get(0); newVolumeId = getVolumeUuidFromPath(volumePath, volumePool); Long size = command.getRestoreVolumeSizes().get(0); - restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile, size, - new Pair<>(vmName, command.getVmState()), mountDirectory, timeout); + restoreVolume(storagePoolMgr, volumePool, volumePath, diskType, backupFile, size, + new Pair<>(vmName, command.getVmState()), mountDirectory, layout, timeout, mountTimeout); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupFiles, mountDirectory, layout, timeout, mountTimeout); } else { - restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupFiles, mountDirectory, layout, timeout, mountTimeout); } } catch (CloudRuntimeException e) { String errorMessage = e.getMessage() != null ? e.getMessage() : ""; @@ -117,18 +181,30 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser return new BackupAnswer(command, true, newVolumeId); } - private void verifyBackupFile(String backupPath, String volUuid) { - if (!checkBackupPathExists(backupPath)) { - throw new CloudRuntimeException(String.format("Backup file for the volume [%s] does not exist.", volUuid)); + /** + * Check every file the volume will be rebuilt from. qcow2 leaves are additionally run through + * {@code qemu-img check}; an {@code rbd export-diff} stream is not a disk image, so there the + * check is existence only — a truncated or corrupt diff is caught by {@code rbd import-diff}, + * which refuses to apply it. + */ + private void verifyBackupFiles(List backupFiles, BackupLayout layout, String volUuid) { + for (String file : backupFiles) { + if (!checkBackupPathExists(file)) { + throw new CloudRuntimeException(String.format("Backup file for the volume [%s] does not exist: %s", volUuid, file)); + } } - if (!checkBackupFileImage(backupPath)) { + if (layout.isRbdDiff()) { + return; + } + String leaf = backupFiles.get(backupFiles.size() - 1); + if (!checkBackupFileImage(leaf)) { throw new CloudRuntimeException(String.format("Backup qcow2 file for the volume [%s] is corrupt.", volUuid)); } } private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, List restoreVolumePools, List restoreVolumePaths, List backedVolumesUUIDs, - String backupPath, List backupFiles, String mountDirectory, int timeout) { + List backupFiles, String mountDirectory, BackupLayout layout, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { @@ -136,51 +212,52 @@ private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, Li String restoreVolumePath = restoreVolumePaths.get(idx); String backupFile = backupFiles.get(idx); String backupVolumeUuid = backedVolumesUUIDs.get(idx); - String fullPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); + List volumeBackupFiles = layout.filesFor(mountDirectory, backupFile, diskType); diskType = "datadisk"; - verifyBackupFile(fullPath, backupVolumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, restoreVolumePool, restoreVolumePath, fullPath, timeout)) { + verifyBackupFiles(volumeBackupFiles, layout, backupVolumeUuid); + if (!replaceVolumeWithBackup(storagePoolMgr, restoreVolumePool, restoreVolumePath, volumeBackupFiles, layout, timeout)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", backupVolumeUuid)); } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, List volumePools, - List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout) { + List volumePaths, List backupFiles, String mountDirectory, + BackupLayout layout, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { PrimaryDataStoreTO volumePool = volumePools.get(i); String volumePath = volumePaths.get(i); String backupFile = backupFiles.get(i); - String bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); + List volumeBackupFiles = layout.filesFor(mountDirectory, backupFile, diskType); String volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); diskType = "datadisk"; - verifyBackupFile(bkpPath, volumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout)) { + verifyBackupFiles(volumeBackupFiles, layout, volumeUuid); + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, volumeBackupFiles, layout, timeout)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid)); } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } - private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile, - Long size, Pair vmNameAndState, String mountDirectory, int timeout) { - String bkpPath; + private void restoreVolume(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile, + Long size, Pair vmNameAndState, String mountDirectory, BackupLayout layout, int timeout, Integer mountTimeout) { + List volumeBackupFiles; String volumeUuid; try { - bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); + volumeBackupFiles = layout.filesFor(mountDirectory, backupFile, diskType); volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); - verifyBackupFile(bkpPath, volumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true, size)) { + verifyBackupFiles(volumeBackupFiles, layout, volumeUuid); + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, volumeBackupFiles, layout, timeout, true, size)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid)); } @@ -190,7 +267,7 @@ private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPa } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } @@ -206,6 +283,7 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT logger.error("Failed to create the tmp mount directory {} for restore", mountDirectory, e); throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } + int exitValue; try { String mountPath = Script.getExecutableAbsolutePath("mount"); List mountCmd = new ArrayList<>(); @@ -226,23 +304,42 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT mountCmd.add("-o"); mountCmd.add(mountOptions); } - Script.executeCommand(mountCmd.toArray(new String[0])); + exitValue = Script.executeCommandForExitValue(mountTimeout, mountCmd.toArray(new String[0])); } catch (Exception e) { logger.error("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory, e); throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); } + if (exitValue != 0) { + logger.error("Failed to mount repository {} of type {} to the directory {}, mount exited with {}", backupRepoAddress, + backupRepoType, mountDirectory, exitValue); + removeTemporaryDirectoryQuietly(mountDirectory); + throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); + } return mountDirectory; } - private void unmountBackupDirectory(String backupDirectory) { + private void unmountBackupDirectory(String backupDirectory, Integer mountTimeout) { + int exitValue; try { String umountPath = Script.getExecutableAbsolutePath("umount"); String[] umountCmd = new String[] { "sudo", umountPath, backupDirectory }; - Script.executeCommand(umountCmd); + exitValue = Script.executeCommandForExitValue(mountTimeout, umountCmd); } catch (Exception e) { logger.error("Failed to unmount backup directory {}", backupDirectory, e); throw new CloudRuntimeException("Failed to unmount the backup directory"); } + if (exitValue != 0) { + logger.error("Failed to unmount backup directory {}, umount exited with {}", backupDirectory, exitValue); + throw new CloudRuntimeException("Failed to unmount the backup directory"); + } + } + + private void removeTemporaryDirectoryQuietly(String backupDirectory) { + try { + Files.deleteIfExists(Paths.get(backupDirectory)); + } catch (IOException e) { + logger.warn("Failed to remove the temporary mount directory {} after the mount failed.", backupDirectory, e); + } } private void deleteTemporaryDirectory(String backupDirectory) { @@ -254,13 +351,6 @@ private void deleteTemporaryDirectory(String backupDirectory) { } } - private String getBackupPath(String mountDirectory, String backupPath, String backupFile, String diskType) { - String bkpPath = String.format(FILE_PATH_PLACEHOLDER, mountDirectory, backupPath); - String backupFileName = String.format("%s.%s.qcow2", diskType.toLowerCase(Locale.ROOT), backupFile); - bkpPath = String.format(FILE_PATH_PLACEHOLDER, bkpPath, backupFileName); - return bkpPath; - } - private boolean checkBackupFileImage(String backupPath) { int exitValue = Script.runSimpleBashScriptForExitValue(String.format("qemu-img check %s", backupPath)); return exitValue == 0; @@ -271,11 +361,18 @@ private boolean checkBackupPathExists(String backupPath) { return exitValue == 0; } - private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout) { - return replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, false, null); + private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, + List backupFiles, BackupLayout layout, int timeout) { + return replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupFiles, layout, timeout, false, null); } - private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume, Long size) { + private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, + List backupFiles, BackupLayout layout, int timeout, boolean createTargetVolume, Long size) { + if (layout.isRbdDiff()) { + return restoreRbdImageFromDiffChain(storagePoolMgr, volumePool, volumePath, backupFiles, size); + } + + String backupPath = backupFiles.get(backupFiles.size() - 1); if (List.of(Storage.StoragePoolType.RBD, Storage.StoragePoolType.Linstor).contains(volumePool.getPoolType())) { return replaceBlockDeviceWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, createTargetVolume, size); } @@ -293,7 +390,7 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr } String[] rsyncCmd = new String[] { Script.getExecutableAbsolutePath("rsync"), "-az", backupPath, volumePath }; - int exitValue = Script.executeCommandForExitValue(rsyncCmd); + int exitValue = Script.executeCommandForExitValue(timeout, rsyncCmd); return exitValue == 0; } @@ -360,40 +457,276 @@ private boolean replaceBlockDeviceWithBackup(KVMStoragePoolManager storagePoolMg return true; } + /** + * Rebuild an RBD image by replaying a chain of {@code rbd export-diff} streams into it: the + * chain's full anchor first, then each incremental in the order it was taken. + * + *

RBD images are raw and cannot hold a backing-file reference, so — unlike the qcow2 path — + * a diff carries no pointer to the one before it and the chain cannot be flattened by + * {@code qemu-img}. Ceph applies the deltas natively instead: each diff ends at the snapshot + * the backup was taken from, {@code import-diff} recreates that snapshot on the target, and the + * next diff in the chain is validated against it. A truncated or out-of-order diff is therefore + * rejected by Ceph rather than silently producing a half-restored image.

+ * + *

The image is recreated first. A full diff only carries the extents that were allocated + * when it was taken, so any block still allocated in the current image would otherwise show + * through the restore as stale data. Recreating the image also drops every RBD snapshot it + * carried: a restore rewinds the volume wholesale, and snapshots of content that is being + * replaced cannot be kept. The chain anchors this path takes are ours to drop, but a snapshot + * taken outside it — a CloudStack volume snapshot kept on primary storage, say — is not, so an + * image carrying one is refused rather than rewound.

+ */ + private boolean restoreRbdImageFromDiffChain(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, + String volumePath, List diffFiles, Long size) { + if (!Storage.StoragePoolType.RBD.equals(volumePool.getPoolType())) { + logger.error("Backup [{}] holds RBD diffs but volume [{}] is on {} storage; it cannot be restored there.", + diffFiles.get(0), volumePath, volumePool.getPoolType()); + return false; + } + KVMStoragePool pool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid()); + Path keyFile = null; + try { + keyFile = writeCephKeyFile(pool.getAuthSecret()); + List connection = rbdConnectionArgs(pool, keyFile); + // Volume paths of RBD volumes are already "/". + String image = volumePath; + + // Prefer the current image's own size: it is what the VM's disk was sized to. Only a + // restore that creates a brand new volume has no image to read it from. + Long imageSize = rbdImageSize(connection, image); + if (imageSize == null) { + imageSize = size; + } + if (imageSize == null || imageSize <= 0) { + logger.error("Unable to determine the size to recreate RBD image {} with", image); + return false; + } + + if (!recreateRbdImage(connection, image, imageSize, rbdDataPool(pool))) { + return false; + } + + for (String diff : diffFiles) { + logger.debug("Applying backup diff {} to RBD image {}", diff, image); + if (Script.executeCommandForExitValue(rbdCommand(connection, "import-diff", diff, image)) != 0) { + logger.error("Failed to apply backup diff {} to RBD image {}", diff, image); + return false; + } + } + + // The snapshots import-diff recreated only existed to chain the diffs together; the + // restored volume should look like any other, so drop them. + if (Script.executeCommandForExitValue(rbdCommand(connection, "snap", "purge", image)) != 0) { + logger.warn("Failed to purge the snapshots left behind by the restore of RBD image {}", image); + } + logger.debug("Successfully restored RBD image {} from {} backup diff(s)", image, diffFiles.size()); + return true; + } catch (IOException e) { + logger.error("Failed to prepare the Ceph credentials needed to restore volume [{}]", volumePath, e); + return false; + } finally { + deleteCephKeyFile(keyFile); + } + } + + /** + * Hand the cephx secret to the {@code rbd} CLI through a 0600 temporary file rather than an + * argument, so it never shows up in the host's process list. + */ + private Path writeCephKeyFile(String authSecret) throws IOException { + if (StringUtils.isBlank(authSecret)) { + return null; + } + Path keyFile = Files.createTempFile("csrestore-ceph-", ".key", + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------"))); + Files.write(keyFile, authSecret.getBytes(StandardCharsets.UTF_8)); + return keyFile; + } + + private void deleteCephKeyFile(Path keyFile) { + if (keyFile == null) { + return; + } + try { + Files.deleteIfExists(keyFile); + } catch (IOException e) { + logger.warn("Failed to delete the temporary Ceph key file [{}].", keyFile, e); + } + } + + private List rbdConnectionArgs(KVMStoragePool pool, Path keyFile) { + List args = new ArrayList<>(); + if (StringUtils.isNotBlank(pool.getSourceHost())) { + args.add("-m"); + args.add(RbdUtils.getRbdMonitors(pool.getSourceHost(), pool.getSourcePort())); + } + if (StringUtils.isNotBlank(pool.getAuthUserName())) { + args.add("--id"); + args.add(pool.getAuthUserName()); + if (keyFile != null) { + args.add("--keyfile"); + args.add(keyFile.toString()); + } + } + return args; + } + + private String[] rbdCommand(List connection, String... arguments) { + List command = new ArrayList<>(); + command.add(Script.getExecutableAbsolutePath("rbd")); + command.addAll(connection); + command.addAll(Arrays.asList(arguments)); + return command.toArray(new String[0]); + } + + /** Size of {@code image} in bytes, or {@code null} when the image does not exist. */ + private Long rbdImageSize(List connection, String image) { + Pair result = Script.executePipedCommands( + Collections.singletonList(rbdCommand(connection, "info", image, "--format", "json")), 0); + if (result.first() == null || result.first() != 0 || result.second() == null) { + return null; + } + Matcher matcher = RBD_INFO_SIZE.matcher(result.second()); + return matcher.find() ? Long.valueOf(matcher.group(1)) : null; + } + + private boolean recreateRbdImage(List connection, String image, long sizeBytes, String dataPool) { + // `rbd rm` refuses to delete an image that still has snapshots, and a restore target + // normally carries the anchor snapshot of the backup chain it belongs to. Ask first, so an + // image without snapshots is not handed a purge with nothing to do; when the listing itself + // fails we cannot tell, and purging is what we would have done anyway. + List snapshots = rbdSnapshots(connection, image); + failIfImageCarriesForeignSnapshots(image, snapshots); + if (snapshots == null || CollectionUtils.isNotEmpty(snapshots)) { + Script.executeCommandForExitValue(rbdCommand(connection, "snap", "purge", image)); + } + if (Script.executeCommandForExitValue(rbdCommand(connection, "rm", image)) != 0 + && rbdImageSize(connection, image) != null) { + logger.error("Failed to remove RBD image {} before restoring it", image); + return false; + } + List create = new ArrayList<>(Arrays.asList("create", image, "--size", sizeBytes + "B")); + for (String feature : RBD_IMAGE_FEATURES) { + create.add("--image-feature"); + create.add(feature); + } + if (StringUtils.isNotBlank(dataPool)) { + create.add("--data-pool"); + create.add(dataPool); + } + if (Script.executeCommandForExitValue(rbdCommand(connection, create.toArray(new String[0]))) != 0) { + logger.error("Failed to recreate RBD image {} with size {} bytes", image, sizeBytes); + return false; + } + return true; + } + + /** The snapshots an image still carries, for explaining why it could not be removed. */ + /** + * Stop before a restore removes a snapshot that this backup path did not take. The chain + * anchors it does take are its own to drop, but a CloudStack volume snapshot kept on primary + * storage, or anything else taken outside it, is content someone asked to keep: losing it is + * not something a restore should do quietly. + */ + private void failIfImageCarriesForeignSnapshots(String image, List snapshots) { + if (snapshots == null) { + return; + } + List foreign = snapshots.stream() + .filter(snapshot -> !snapshot.startsWith(RBD_BACKUP_SNAPSHOT_PREFIX)) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(foreign)) { + throw new CloudRuntimeException(String.format("RBD image %s carries snapshot(s) [%s] which this backup " + + "path did not take. Restoring the volume rewinds it wholesale and would remove them, so it is " + + "refused; delete those snapshots first, or restore into a new volume instead.", + image, String.join(", ", foreign))); + } + } + + /** The snapshots an image carries, or {@code null} when they could not be listed at all. */ + private List rbdSnapshots(List connection, String image) { + Pair result = Script.executePipedCommands( + Collections.singletonList(rbdCommand(connection, "snap", "ls", image, "--format", "json")), 0); + if (result.first() == null || result.first() != 0 || StringUtils.isBlank(result.second())) { + return null; + } + List names = new ArrayList<>(); + Matcher matcher = RBD_SNAP_NAME.matcher(result.second()); + while (matcher.find()) { + names.add(matcher.group(1)); + } + return names; + } + + private String rbdDataPool(KVMStoragePool pool) { + Map details = pool.getDetails(); + return details == null ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); + } + private boolean attachVolumeToVm(KVMStoragePoolManager storagePoolMgr, String vmName, PrimaryDataStoreTO volumePool, String volumePath) { String deviceToAttachDiskTo = getDeviceToAttachDisk(vmName); + if (Storage.StoragePoolType.RBD.equals(volumePool.getPoolType())) { + return attachRbdVolumeToVm(storagePoolMgr, vmName, volumePool, volumePath, deviceToAttachDiskTo); + } List virshCmd = new ArrayList<>(); virshCmd.add(Script.getExecutableAbsolutePath("virsh")); - if (volumePool.getPoolType() == Storage.StoragePoolType.RBD) { - String xmlForRbdDisk = getXmlForRbdDisk(storagePoolMgr, volumePool, volumePath, deviceToAttachDiskTo); - logger.debug("RBD disk xml to attach: {}", xmlForRbdDisk); - virshCmd.add("attach-device"); - virshCmd.add(vmName); - virshCmd.add("/dev/stdin"); - virshCmd.add("< result = Script.executePipedCommands(Arrays.asList(domblkCmd, tailCmd, headCmd, awkCmd), 0); - String currentDevice = result.second(); + // executePipedCommands appends a line separator to every line it reads, so the device + // name has to be trimmed before the last character can be incremented. + String currentDevice = result.second() == null ? "" : result.second().trim(); + if (result.first() == null || result.first() != 0 || StringUtils.isBlank(currentDevice)) { + throw new CloudRuntimeException(String.format("Failed to determine the device to attach the restored volume to on VM [%s].", vmName)); + } char lastChar = currentDevice.charAt(currentDevice.length() - 1); char incrementedChar = (char) (lastChar + 1); return currentDevice.substring(0, currentDevice.length() - 1) + incrementedChar; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java index 507744fdc316..e27b8460f624 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java @@ -51,6 +51,7 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.RbdUtils; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @@ -96,7 +97,7 @@ public Answer execute(final RevertSnapshotCommand command, final LibvirtComputin if (primaryPool.getType() == StoragePoolType.RBD) { Rados rados = new Rados(primaryPool.getAuthUserName()); - rados.confSet(MON_HOST, primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + rados.confSet(MON_HOST, RbdUtils.getRbdMonitors(primaryPool.getSourceHost(), primaryPool.getSourcePort())); rados.confSet(KEY, primaryPool.getAuthSecret()); rados.confSet(CLIENT_MOUNT_TIMEOUT, RADOS_CONNECTION_TIMEOUT); rados.connect(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 106fe31a0f18..9588f2fb2257 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -19,7 +19,6 @@ package com.cloud.hypervisor.kvm.resource.wrapper; -import com.amazonaws.util.CollectionUtils; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; @@ -33,6 +32,8 @@ import org.apache.cloudstack.backup.BackupAnswer; import org.apache.cloudstack.backup.TakeBackupCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Arrays; @@ -52,6 +53,13 @@ public class LibvirtTakeBackupCommandWrapper extends CommandWrapper runBackupScript(LibvirtComputingResource libvirtCo "-m", Objects.nonNull(mountOptions) ? mountOptions : "", "-p", backupPath, "-q", command.getQuiesce() != null && command.getQuiesce() ? "true" : "false", - "-d", diskPaths.isEmpty() ? "" : String.join(",", diskPaths) + "-d", CollectionUtils.isEmpty(diskPaths) ? "" : String.join(",", diskPaths) )); - if (mode != null && !mode.isEmpty()) { + if (StringUtils.isNotBlank(mode)) { argv.add("-M"); argv.add(mode); } - if (bitmapNew != null && !bitmapNew.isEmpty()) { + if (StringUtils.isNotBlank(command.getMethod())) { + argv.add("--method"); + argv.add(command.getMethod()); + } + if (StringUtils.isNotBlank(bitmapNew)) { argv.add("--bitmap-new"); argv.add(bitmapNew); } - if (bitmapParent != null && !bitmapParent.isEmpty()) { + if (StringUtils.isNotBlank(bitmapParent)) { argv.add("--bitmap-parent"); argv.add(bitmapParent); } - if (parentPaths != null && !parentPaths.isEmpty()) { + if (CollectionUtils.isNotEmpty(parentPaths)) { argv.add("--parent-paths"); argv.add(String.join(",", parentPaths)); } @@ -181,48 +193,86 @@ private Pair runBackupScript(LibvirtComputingResource libvirtCo */ private String validateBackupArgs(TakeBackupCommand command) { String mode = command.getMode(); - if (mode == null || mode.isEmpty()) { - return null; // legacy full-only — no extra args expected + String method = StringUtils.defaultIfBlank(command.getMethod(), METHOD_QCOW2); + if (METHOD_RBD.equals(method)) { + return validateRbdBackupArgs(command, mode); + } + if (!METHOD_QCOW2.equals(method)) { + return "Unknown backup method: " + method; + } + if (StringUtils.isBlank(mode)) { + return null; // legacy full-only, no extra args expected } if (MODE_INCREMENTAL.equals(mode)) { - if (command.getBitmapNew() == null || command.getBitmapNew().isEmpty()) { + if (StringUtils.isBlank(command.getBitmapNew())) { return "incremental mode requires bitmapNew"; } - if (command.getBitmapParent() == null || command.getBitmapParent().isEmpty()) { + if (StringUtils.isBlank(command.getBitmapParent())) { return "incremental mode requires bitmapParent"; } - if (command.getParentPaths() == null || command.getParentPaths().isEmpty()) { + if (CollectionUtils.isEmpty(command.getParentPaths())) { return "incremental mode requires parentPaths"; } return null; } if (MODE_FULL.equals(mode)) { - if (command.getBitmapNew() == null || command.getBitmapNew().isEmpty()) { + if (StringUtils.isBlank(command.getBitmapNew())) { return "full mode requires bitmapNew (the bitmap to create for the next incremental)"; } return null; } if (MODE_LEGACY_FULL.equals(mode)) { - return null; // feature-off full backup — no bitmap or chain args expected + return null; // feature-off full backup, no bitmap or chain args expected } return "Unknown backup mode: " + mode; } + /** + * An RBD chain anchors each cycle on an {@code rbd snap} whose name travels in bitmapNew, and + * an incremental diffs it against the previous anchor in bitmapParent. No parent file paths: + * the diff stream is replayed on restore rather than rebased onto its parent. + */ + private String validateRbdBackupArgs(TakeBackupCommand command, String mode) { + if (!MODE_FULL.equals(mode) && !MODE_INCREMENTAL.equals(mode)) { + return "Unknown backup mode for the rbd method: " + mode; + } + if (command.getBitmapNew() == null || command.getBitmapNew().isEmpty()) { + return mode + " mode requires bitmapNew (the RBD snapshot to create as the chain anchor)"; + } + if (MODE_INCREMENTAL.equals(mode) + && (command.getBitmapParent() == null || command.getBitmapParent().isEmpty())) { + return "incremental mode requires bitmapParent (the RBD snapshot to diff against)"; + } + if (command.getVolumePaths() == null || command.getVolumePaths().isEmpty()) { + return mode + " mode requires the VM's volume paths"; + } + return null; + } + /** * Sum the per-disk size lines emitted by nasbackup.sh. Single-volume mode emits one * line containing just the byte count; multi-volume mode emits one line per disk * whose first whitespace-separated token is the byte count. + * + *

Lines that do not start with a number are skipped rather than aborting the parse: the + * script's stdout also carries whatever the mount produced, and losing an otherwise complete + * backup over a stray warning line would be worse than reporting a size of 0.

*/ private long parseBackupSize(String stdout, List diskPaths) { long backupSize = 0L; - if (CollectionUtils.isNullOrEmpty(diskPaths)) { + if (CollectionUtils.isEmpty(diskPaths)) { List outputLines = Arrays.asList(stdout.split("\n")); if (!outputLines.isEmpty()) { backupSize = Long.parseLong(outputLines.get(outputLines.size() - 1).trim()); } } else { for (String line : stdout.split("\n")) { - backupSize = backupSize + Long.parseLong(line.split(" ")[0].trim()); + String sizeToken = line.trim().split("\\s+")[0]; + if (!sizeToken.matches("\\d+")) { + logger.debug("Ignoring non-size line in nasbackup.sh output: {}", line); + continue; + } + backupSize = backupSize + Long.parseLong(sizeToken); } } return backupSize; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java index 8a9d69c97954..2cc650520c49 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import com.cloud.utils.RbdUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; @@ -25,11 +26,8 @@ import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; -import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; public class KVMPhysicalDisk { @@ -51,12 +49,12 @@ public static String RBDStringBuilder(KVMStoragePool storagePool, String image) String dataPool = (details == null) ? null : details.get(RBD_DEFAULT_DATA_POOL); String rbdOpts = "rbd:" + image; - rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort); + rbdOpts += ":mon_host=" + RbdUtils.composeOptionForMonHosts(monHost, monPort); if (authUserName == null) { - rbdOpts += ":auth_supported=none"; + rbdOpts += ":auth_client_required=none"; } else { - rbdOpts += ":auth_supported=cephx"; + rbdOpts += ":auth_client_required=cephx"; rbdOpts += ":id=" + authUserName; rbdOpts += ":key=" + authSecret; } @@ -71,25 +69,6 @@ public static String RBDStringBuilder(KVMStoragePool storagePool, String image) return rbdOpts; } - private static String composeOptionForMonHosts(String monHost, int monPort) { - List hosts = new ArrayList<>(); - for (String host : monHost.split(",")) { - if (monPort > 0) { - hosts.add(replaceHostAddress(host) + "\\:" + monPort); - } else { - hosts.add(replaceHostAddress(host)); - } - } - return StringUtils.join(hosts, "\\;"); - } - - private static String replaceHostAddress(String hostIp) { - if (hostIp != null && hostIp.startsWith("[") && hostIp.endsWith("]")) { - return hostIp.replaceAll("\\:", "\\\\:"); - } - return hostIp; - } - public static long getVirtualSizeFromFile(String path) { try { QemuImg qemu = new QemuImg(0); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index a8207cec3fa6..260ffbe8d633 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -126,4 +126,8 @@ default LibvirtVMDef.DiskDef.BlockIOSize getSupportedPhysicalBlockSize() { default void customizeLibvirtDiskDef(LibvirtVMDef.DiskDef disk) { } + + default String getAuthSecretUUID() { + return null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index d99847fd921a..c56171577e09 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -37,6 +37,7 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.kvm.resource.KVMHABase; import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; @@ -445,7 +446,7 @@ public boolean disconnectPhysicalDisk(StoragePoolType type, String poolUuid, Str public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) { StorageAdaptor adaptor = getStorageAdaptor(type); - if (type == StoragePoolType.NetworkFilesystem) { + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) { _haMonitor.removeStoragePool(uuid); } boolean deleteStatus = adaptor.deleteStoragePool(uuid);; @@ -457,7 +458,7 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); - if (type == StoragePoolType.NetworkFilesystem) { + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) { _haMonitor.removeStoragePool(uuid); } boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 11acb9546b53..b4eea8bdd901 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -156,6 +156,7 @@ import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateLocation; +import com.cloud.utils.RbdUtils; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; @@ -182,6 +183,12 @@ public class KVMStorageProcessor implements StorageProcessor { private static final String CEPH_AUTH_KEY = "key"; private static final String CEPH_CLIENT_MOUNT_TIMEOUT = "client_mount_timeout"; private static final String CEPH_DEFAULT_MOUNT_TIMEOUT = "30"; + + // libvirt < 10.1.0 has an object apply-order bug (fixed in 10.1.0) that breaks hot-plug of an encrypted + // blockdev: on attach the disk is opened before its LUKS secret object is defined, so the attach fails with + // "No secret with id '...-format-encryption-secret0'". Booting a VM from an encrypted disk is unaffected (the + // QEMU command line resolves all -object before -blockdev). See qemuBlockStorageSourceAttachApply() in libvirt. + private static final long MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG = 10001000L; // libvirt 10.1.0 /** * Time interval before rechecking virsh commands */ @@ -1795,6 +1802,20 @@ protected DiskDef.DiskBus getAttachDiskBusType(int deviceId, List disks return DiskDef.DiskBus.VIRTIO; } + /** + * libvirt < 10.1.0 cannot hot-plug an encrypted rbd blockdev (the LUKS secret is applied after the disk is + * opened), so refuse the attach with a clear message rather than letting libvirt fail with an opaque + * "No secret with id ..." error. Only the RBD hot-plug path is affected; booting a VM from an encrypted RBD + * disk works on older libvirt, so this does not gate the boot/root path. + */ + protected void ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType poolType) { + if (poolType == StoragePoolType.RBD + && resource.getHypervisorLibvirtVersion() < MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG) { + throw new CloudRuntimeException("Libvirt version 10.1.0 required to attach an encrypted RBD volume to a running VM, but version " + + resource.getHypervisorLibvirtVersion() + " detected. Booting a VM from an encrypted RBD disk is not affected."); + } + } + @Override public Answer attachVolume(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); @@ -1807,8 +1828,13 @@ public Answer attachVolume(final AttachCommand cmd) { final Connect conn = LibvirtConnection.getConnectionByVmName(vmName); DiskDef.LibvirtDiskEncryptDetails encryptDetails = null; if (vol.requiresEncryption()) { + // Encrypted RBD is decrypted by librbd inside qemu; hot-plugging it needs a libvirt new enough to + // emit the LUKS secret before the rbd blockdev. Booting from an encrypted RBD disk is unaffected. + ensureLibvirtSupportsEncryptedRbdHotplug(primaryStore.getPoolType()); String secretUuid = resource.createLibvirtVolumeSecret(conn, vol.getPath(), vol.getPassphrase()); - encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat())); + // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. + String encryptEngine = (primaryStore.getPoolType() == StoragePoolType.RBD) ? "librbd" : null; + encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat()), encryptEngine); vol.clearPassphrase(); } @@ -2568,33 +2594,36 @@ private CreateObjectAnswer takeClvmVolumeSnapshotOfStoppedVm(KVMPhysicalDisk dis * barriers properly (>2.6.32) this won't be any different then pulling the power * cord out of a running machine. */ - private Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + protected Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { Long snapshotSize = null; + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage image = null; try { - Rados r = radosConnect(primaryPool); + r = radosConnect(primaryPool); - final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); - final Rbd rbd = new Rbd(io); - final RbdImage image = rbd.open(disk.getName()); + io = r.ioCtxCreate(primaryPool.getSourceDir()); + rbd = new Rbd(io); + image = rbd.open(disk.getName()); logger.debug("Attempting to create RBD snapshot {}@{}", disk.getName(), snapshotName); image.snapCreate(snapshotName); - image.snapCreate(snapshotName); long rbdSnapshotSize = getRbdSnapshotSize(primaryPool.getSourceDir(), disk.getName(), snapshotName, primaryPool.getSourceHost(), primaryPool.getAuthUserName(), primaryPool.getAuthSecret()); if (rbdSnapshotSize > 0) { snapshotSize = rbdSnapshotSize; } - - rbd.close(image); - r.ioCtxDestroy(io); } catch (final Exception e) { logger.error("A RBD snapshot operation on [{}] failed. The error was: {}", disk.getName(), e.getMessage(), e); + } finally { + closeRbdImage(rbd, image, disk.getName()); + destroyRadosIoCtx(r, io, disk.getName()); } return snapshotSize; } - private long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { + protected long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { logger.debug("Get RBD snapshot size for {}/{}@{}", poolPath, diskName, snapshotName); //cmd: rbd du /@ --format json --mon-host --id --key 2>/dev/null String snapshotDetailsInJson = Script.runSimpleBashScript(String.format("rbd du %s/%s@%s --format json --mon-host %s --id %s --key %s 2>/dev/null", poolPath, diskName, snapshotName, rbdMonitor, authUser, authSecret)); @@ -2881,9 +2910,9 @@ protected boolean isAvailablePoolSizeDividedByDiskSizeLesserThanMinRate(long ava return ((availablePoolSize * 1d) / (diskSize * 1d)) < MIN_RATE_BETWEEN_AVAILABLE_POOL_AND_DISK_SIZE_TO_TAKE_DISK_SNAPSHOT; } - private Rados radosConnect(final KVMStoragePool primaryPool) throws RadosException { + protected Rados radosConnect(final KVMStoragePool primaryPool) throws RadosException { Rados r = new Rados(primaryPool.getAuthUserName()); - r.confSet(CEPH_MON_HOST, primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); + r.confSet(CEPH_MON_HOST, RbdUtils.getRbdMonitors(primaryPool.getSourceHost(), primaryPool.getSourcePort())); r.confSet(CEPH_AUTH_KEY, primaryPool.getAuthSecret()); r.confSet(CEPH_CLIENT_MOUNT_TIMEOUT, CEPH_DEFAULT_MOUNT_TIMEOUT); r.connect(); @@ -2891,6 +2920,50 @@ private Rados radosConnect(final KVMStoragePool primaryPool) throws RadosExcepti return r; } + /** + * Closes an RBD image if it was opened; never throws. An image left open keeps this client's RBD + * exclusive-lock, which later makes 'rbd snap rollback' (revertSnapshot) fail with EROFS and keeps + * the image busy so it cannot be removed. + */ + protected void closeRbdImage(Rbd rbd, RbdImage image, String imageName) { + if (image == null) { + return; + } + try { + rbd.close(image); + } catch (final Exception e) { + logger.warn("Failed to close RBD image [{}]. The error was: {}", imageName, e.getMessage(), e); + } + } + + /** Destroys a RADOS IO context if it was created; never throws. */ + protected void destroyRadosIoCtx(Rados r, IoCTX io, String contextDescription) { + if (io == null) { + return; + } + try { + r.ioCtxDestroy(io); + } catch (final Exception e) { + logger.warn("Failed to destroy the RADOS IO context used for [{}]. The error was: {}", contextDescription, e.getMessage(), e); + } + } + + /** + * Unprotects an RBD snapshot if it was protected; never throws. A snapshot left protected cannot + * be deleted, and neither can its volume. + */ + protected void unprotectRbdSnapshot(RbdImage image, String snapshotName, boolean snapProtected) { + if (!snapProtected) { + return; + } + try { + image.snapUnprotect(snapshotName); + } catch (final Exception e) { + logger.error("Failed to unprotect RBD snapshot [{}]; it and its volume cannot be deleted until this is resolved manually. The error was: {}", + snapshotName, e.getMessage(), e); + } + } + @Override public Answer deleteVolume(final DeleteCommand cmd) { final VolumeObjectTO vol = (VolumeObjectTO)cmd.getData(); @@ -3046,17 +3119,24 @@ private KVMPhysicalDisk createRBDvolumeFromRBDSnapshot(KVMPhysicalDisk volume, S disk.setSize(size > volume.getVirtualSize() ? size : volume.getVirtualSize()); disk.setVirtualSize(size > volume.getVirtualSize() ? size : disk.getSize()); + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage srcImage = null; + RbdImage diskImage = null; + boolean snapProtected = false; + try { - Rados r = new Rados(srcPool.getAuthUserName()); - r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); + r = new Rados(srcPool.getAuthUserName()); + r.confSet("mon_host", RbdUtils.getRbdMonitors(srcPool.getSourceHost(), srcPool.getSourcePort())); r.confSet("key", srcPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); r.connect(); - IoCTX io = r.ioCtxCreate(srcPool.getSourceDir()); - Rbd rbd = new Rbd(io); - RbdImage srcImage = rbd.open(volume.getName()); + io = r.ioCtxCreate(srcPool.getSourceDir()); + rbd = new Rbd(io); + srcImage = rbd.open(volume.getName()); List snaps = srcImage.snapList(); boolean snapFound = false; @@ -3072,23 +3152,26 @@ private KVMPhysicalDisk createRBDvolumeFromRBDSnapshot(KVMPhysicalDisk volume, S return null; } srcImage.snapProtect(snapshotName); + snapProtected = true; logger.debug(String.format("Try to clone snapshot %s on RBD", snapshotName)); rbd.clone(volume.getName(), snapshotName, io, disk.getName(), LibvirtStorageAdaptor.RBD_FEATURES, 0); - RbdImage diskImage = rbd.open(disk.getName()); + diskImage = rbd.open(disk.getName()); if (disk.getVirtualSize() > volume.getVirtualSize()) { diskImage.resize(disk.getVirtualSize()); } diskImage.flatten(); - rbd.close(diskImage); - - srcImage.snapUnprotect(snapshotName); - rbd.close(srcImage); - r.ioCtxDestroy(io); } catch (RadosException | RbdException e) { logger.error(String.format("Failed due to %s", e.getMessage()), e); disk = null; + } finally { + // Every handle has to be released on all paths, including the "snapshot not found" return and + // any failure of clone/resize/flatten. + closeRbdImage(rbd, diskImage, newUuid); + unprotectRbdSnapshot(srcImage, snapshotName, snapProtected); + closeRbdImage(rbd, srcImage, volume.getName()); + destroyRadosIoCtx(r, io, snapshotName); } return disk; @@ -3406,7 +3489,8 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { newVol.setPath(path); ImageFormat newVolumeFormat = getFormat(newVolume.getFormat()); newVol.setFormat(newVolumeFormat); - newVol.setEncryptFormat(destVol.getEncryptFormat()); + // an encrypted volume copied to another kind of pool is encrypted differently (librbd LUKS2 on RBD, qemu LUKS otherwise) + newVol.setEncryptFormat(newVolume.getQemuEncryptFormat() != null ? newVolume.getQemuEncryptFormat().toString() : destVol.getEncryptFormat()); return new CopyCmdAnswer(newVol); } catch (final CloudRuntimeException e) { logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 4bfac31b68f9..6e3c23f2bdee 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -21,8 +21,12 @@ import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,7 +38,9 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; @@ -43,6 +49,7 @@ import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -76,6 +83,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; +import com.cloud.utils.RbdUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import com.cloud.utils.storage.TemplateDownloaderUtil; @@ -95,6 +103,17 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_DEEP_FLATTEN = 32; public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ + /* Space reserved at the front of an encrypted RBD image for the LUKS2 header/keyslots so the + usable (decrypted) size still matches the requested volume size. */ + public static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB + /* libvirt's VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS, not exposed as a constant by libvirt-java */ + private static final int VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS = 2; + + /* pool types an encrypted volume can be copied between, see copyEncryptedPhysicalDisk */ + private static final Set ENCRYPTED_COPY_POOL_TYPES = Set.of(StoragePoolType.RBD, StoragePoolType.NetworkFilesystem, + StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint); + private static final String SRC_LUKS_SECRET_ID = "srcsec0"; + private static final String DEST_LUKS_SECRET_ID = "destsec0"; private static final Set QEMU_IMG_MANAGED_POOL_TYPES = Set.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint); @@ -600,6 +619,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { pool.setAuthSecret(secretValue); } } + pool.setAuthSecretUUID(spd.getSecretUUID()); /** * On large (RBD) storage pools it can take up to a couple of minutes @@ -640,6 +660,19 @@ uuid, toHumanReadableSize(storage.getInfo().capacity), public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + /* + * An RBD volume is looked up through librbd rather than through libvirt. + * A volume that was created by another host is not in the libvirt pool + * cache, so looking it up through libvirt misses and forces a refresh of + * the whole pool. Refreshing an RBD pool stats every image in it, so that + * cost grows with the number of volumes in the pool and is paid on every + * VM start. Every other RBD operation in this class already uses librbd + * directly. + */ + if (pool.getType() == StoragePoolType.RBD) { + return getRbdPhysicalDisk(volumeUuid, libvirtPool); + } + try { StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); if (vol == null) { @@ -681,6 +714,74 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { } } + /** + * Looks an RBD volume up directly through librbd. + * + * The size reported by librbd is used for both the size and the virtual size of + * the disk, matching what this class already does after converting an image into + * an RBD volume. + */ + private KVMPhysicalDisk getRbdPhysicalDisk(String volumeUuid, LibvirtStoragePool pool) { + Rados r = new Rados(pool.getAuthUserName()); + try { + r.confSet("mon_host", RbdUtils.getRbdMonitors(pool.getSourceHost(), pool.getSourcePort())); + /* + * The secret is null when the pool has no cephx user, and librados + * aborts the process rather than returning an error if it is handed a + * null value here. + */ + if (pool.getAuthUserName() != null) { + r.confSet("key", pool.getAuthSecret()); + } else { + r.confSet("auth_client_required", "none"); + } + r.confSet("client_mount_timeout", "30"); + r.connect(); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + Rbd rbd = new Rbd(io); + // The image is only stat'ed, so it is opened read only and cannot take the exclusive lock. + RbdImage image = rbd.openReadOnly(volumeUuid); + try { + RbdImageInfo rbdInfo = image.stat(); + KVMPhysicalDisk disk = new KVMPhysicalDisk(pool.getSourceDir() + "/" + volumeUuid, volumeUuid, pool); + disk.setFormat(PhysicalDiskFormat.RAW); + disk.setSize(rbdInfo.size); + disk.setVirtualSize(rbdInfo.size); + return disk; + } finally { + closeRbdImage(rbd, image, volumeUuid); + } + } finally { + r.ioCtxDestroy(io); + } + } catch (RadosException e) { + logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage() + + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException(e.toString(), e); + } catch (RbdException e) { + logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage() + + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException(e.toString(), e); + } finally { + r.shutDown(); + } + } + + /** + * Closes an RBD image without throwing, so that a failure to close cannot discard a + * successful result or hide the exception that is already on its way out. + */ + private void closeRbdImage(Rbd rbd, RbdImage image, String volumeUuid) { + try { + rbd.close(image); + } catch (RbdException e) { + logger.warn("Failed to close RBD image " + volumeUuid + " (" + e.getReturnValue() + "): " + + e.getMessage() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + } + } + /** * adjust refcount */ @@ -988,8 +1089,18 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); - return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : - createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase); + // Create the raw RBD image first. For encrypted volumes we apply a native librbd LUKS header + // afterwards via `rbd encryption format` (engine='librbd'). We deliberately do NOT hand the + // passphrase to qemu-img, which would instead produce a qemu-native LUKS container. + KVMPhysicalDisk disk = (dataPool == null) ? + createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : + createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, null); + + if (passphrase != null && passphrase.length > 0) { + formatRbdImageEncryption(pool, name, passphrase); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + } + return disk; } else if (QEMU_IMG_MANAGED_POOL_TYPES.contains(poolType)) { switch (format) { case QCOW2: @@ -1150,61 +1261,6 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag logger.info("Attempting to remove volume " + uuid + " from pool " + pool.getUuid()); - /** - * RBD volume can have snapshots and while they exist libvirt - * can't remove the RBD volume - * - * We have to remove those snapshots first - */ - if (pool.getType() == StoragePoolType.RBD) { - try { - logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + " prior to removing the image"); - - Rados r = new Rados(pool.getAuthUserName()); - r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); - r.confSet("key", pool.getAuthSecret()); - r.confSet("client_mount_timeout", "30"); - r.connect(); - logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host")); - - IoCTX io = r.ioCtxCreate(pool.getSourceDir()); - Rbd rbd = new Rbd(io); - RbdImage image = rbd.open(uuid); - logger.debug("Fetching list of snapshots of RBD image " + pool.getSourceDir() + "/" + uuid); - List snaps = image.snapList(); - try { - for (RbdSnapInfo snap : snaps) { - if (image.snapIsProtected(snap.name)) { - logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); - image.snapUnprotect(snap.name); - } else { - logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected."); - } - logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); - image.snapRemove(snap.name); - } - logger.info("Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " - + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); - } catch (RbdException e) { - logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); - } finally { - logger.debug("Closing image and destroying context"); - rbd.close(image); - r.ioCtxDestroy(io); - } - } catch (RadosException e) { - logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); - } catch (RbdException e) { - logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); - } - } - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), uuid); @@ -1225,6 +1281,58 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } } + static String maskRbdKey(String message) { + return message == null ? null : message.replaceAll("key=[^:\\s]*", "key=******"); + } + + private Rados connectToCeph(KVMStoragePool pool) throws RadosException { + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", RbdUtils.getRbdMonitors(pool.getSourceHost(), pool.getSourcePort())); + r.confSet("key", pool.getAuthSecret()); + r.confSet("client_mount_timeout", "30"); + r.connect(); + return r; + } + + private boolean rbdImageExists(KVMStoragePool pool, String imageName) throws RadosException, RbdException { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + return Arrays.asList(new Rbd(io).list()).contains(imageName); + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } + + /** + * Removes an RBD image left behind by a failed copy, so that it does not make every later copy fail with "File exists" + */ + private void removeRbdImageQuietly(KVMStoragePool pool, String imageName) { + try { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + Rbd rbd = new Rbd(io); + if (Arrays.asList(rbd.list()).contains(imageName)) { + rbd.remove(imageName); + logger.info("Removed the incomplete RBD image {}/{} left by the failed copy", pool.getSourceDir(), imageName); + } + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } catch (RadosException | RbdException e) { + logger.warn("Failed to remove the incomplete RBD image {}/{} left by the failed copy, remove it manually: {}", pool.getSourceDir(), imageName, e.getMessage()); + } + } + /** * This function copies a physical disk from Secondary Storage to Primary Storage * or from Primary to Primary Storage @@ -1244,7 +1352,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { - disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); + disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout, passphrase); } else { try (KeyFile keyFile = new KeyFile(passphrase)){ String newUuid = name; @@ -1324,7 +1432,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, } private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){ + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase){ /* With RBD you can't run qemu-img convert with an existing RBD image as destination @@ -1335,6 +1443,8 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ KVMStoragePool srcPool = template.getPool(); + Map destDetails = destPool.getDetails(); + String dataPool = (destDetails == null) ? null : destDetails.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); KVMPhysicalDisk disk = null; String newUuid = name; @@ -1351,6 +1461,16 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } + if (passphrase != null && passphrase.length > 0) { + boolean sameClusterRbd = srcPool.getType() == StoragePoolType.RBD + && srcPool.getSourceHost().equals(destPool.getSourceHost()) + && srcPool.getSourceDir().equals(destPool.getSourceDir()); + if (sameClusterRbd) { + return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase); + } + return createEncryptedRootFullCopy(srcPool, template, destPool, newUuid, disk, passphrase); + } + QemuImgFile srcFile; QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath())); destFile.setFormat(format); @@ -1380,9 +1500,13 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, logger.debug("Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); Rados r = new Rados(srcPool.getAuthUserName()); - r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); + r.confSet("mon_host", RbdUtils.getRbdMonitors(srcPool.getSourceHost(), srcPool.getSourcePort())); r.confSet("key", srcPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); + if (dataPool != null) { + logger.debug("Setting RBD data pool to " + dataPool + " for the new image " + disk.getName()); + r.confSet(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL, dataPool); + } r.connect(); logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host")); @@ -1449,16 +1573,20 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, logger.debug("Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); Rados rSrc = new Rados(srcPool.getAuthUserName()); - rSrc.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); + rSrc.confSet("mon_host", RbdUtils.getRbdMonitors(srcPool.getSourceHost(), srcPool.getSourcePort())); rSrc.confSet("key", srcPool.getAuthSecret()); rSrc.confSet("client_mount_timeout", "30"); rSrc.connect(); logger.debug("Successfully connected to source Ceph cluster at " + rSrc.confGet("mon_host")); Rados rDest = new Rados(destPool.getAuthUserName()); - rDest.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + rDest.confSet("mon_host", RbdUtils.getRbdMonitors(destPool.getSourceHost(), destPool.getSourcePort())); rDest.confSet("key", destPool.getAuthSecret()); rDest.confSet("client_mount_timeout", "30"); + if (dataPool != null) { + logger.debug("Setting RBD data pool to " + dataPool + " on the destination cluster for the new image " + disk.getName()); + rDest.confSet(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL, dataPool); + } rDest.connect(); logger.debug("Successfully connected to source Ceph cluster at " + rDest.confGet("mon_host")); @@ -1493,9 +1621,126 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, disk = null; } } + + // Encrypted volumes are handled by the early return above (create empty -> luks2 format -> + // import template through encryption); the clone/convert path here is for plaintext volumes. + return disk; + } + + /** + * Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster + * as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to + * reserve LUKS2-header space, snapshot+protect that grown state, clone from it, apply a LUKS2 header, + * then resize the clone to the requested size. The inherited (plaintext) template data stays readable + * through the clone's encryption, and the clone is a thin CoW image (only the header is written). + * + * @return the encrypted CoW clone, or {@code null} if the Ceph operations failed + */ + private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KVMStoragePool destPool, + String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + String luksReservedSnapshotName = rbdTemplateSnapName + "-luks"; + Rados radosConnection = null; + IoCTX ioContext = null; + Rbd rbdClient = null; + RbdImage templateImage = null; + try { + radosConnection = new Rados(destPool.getAuthUserName()); + radosConnection.confSet("mon_host", RbdUtils.getRbdMonitors(destPool.getSourceHost(), destPool.getSourcePort())); + radosConnection.confSet("key", destPool.getAuthSecret()); + radosConnection.confSet("client_mount_timeout", "30"); + radosConnection.connect(); + ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); + rbdClient = new Rbd(ioContext); + templateImage = rbdClient.open(template.getName()); + boolean luksSnapshotExists = false; + for (RbdSnapInfo snapshotInfo : templateImage.snapList()) { + if (luksReservedSnapshotName.equals(snapshotInfo.name)) { + luksSnapshotExists = true; + break; + } + } + if (!luksSnapshotExists) { + templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); + templateImage.snapCreate(luksReservedSnapshotName); + templateImage.snapProtect(luksReservedSnapshotName); + logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName); + } + rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage()); + return null; + } finally { + if (rbdClient != null && templateImage != null) { + try { + rbdClient.close(templateImage); + } catch (RbdException ignored) { + // best-effort close of the template handle + } + } + if (radosConnection != null && ioContext != null) { + radosConnection.ioCtxDestroy(ioContext); + } + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + if (disk.getVirtualSize() > template.getVirtualSize()) { + // grow the clone to the requested root size (encryption-aware) + new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(), + destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), + newUuid, disk.getVirtualSize(), false, passphrase); + } + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + + /** + * Option B (full-copy encrypted root), used when the template is not on the same RBD cluster (e.g. first + * use from secondary storage). Create an empty image, apply a LUKS2 header, then import the template + * THROUGH the encryption layer (qemu-img convert -n). Correct but not thin (no CoW). + * + * @return the encrypted image, or {@code null} if the Ceph operations failed + */ + private KVMPhysicalDisk createEncryptedRootFullCopy(KVMStoragePool srcPool, KVMPhysicalDisk template, + KVMStoragePool destPool, String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + long imageSizeWithLuksHeader = disk.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES; + Rados radosConnection = null; + IoCTX ioContext = null; + try { + radosConnection = new Rados(destPool.getAuthUserName()); + radosConnection.confSet("mon_host", RbdUtils.getRbdMonitors(destPool.getSourceHost(), destPool.getSourcePort())); + radosConnection.confSet("key", destPool.getAuthSecret()); + radosConnection.confSet("client_mount_timeout", "30"); + radosConnection.connect(); + ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); + Rbd rbdClient = new Rbd(ioContext); + rbdClient.create(newUuid, imageSizeWithLuksHeader, RBD_FEATURES, rbdOrder); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted RBD image {}: {}", newUuid, e.getMessage()); + return null; + } finally { + if (radosConnection != null && ioContext != null) { + radosConnection.ioCtxDestroy(ioContext); + } + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + boolean sourceIsRbdPool = srcPool.getType() == StoragePoolType.RBD; + new RbdEncryption().importTemplate( + sourceIsRbdPool ? srcPool.getSourceDir() : null, sourceIsRbdPool ? template.getName() : null, + sourceIsRbdPool ? null : template.getPath(), sourceIsRbdPool ? null : template.getFormat().toString(), + destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), + destPool.getSourceDir(), newUuid, passphrase, CryptSetup.LuksType.LUKS2); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); return disk; } + /** + * Apply native librbd LUKS encryption to an existing RBD image via the rbd CLI. + * Isolated here so the CLI dependency can later be swapped for a native (JNA) librbd binding. + */ + private void formatRbdImageEncryption(KVMStoragePool pool, String image, byte[] passphrase) { + new RbdEncryption().format(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getAuthSecret(), pool.getSourceDir(), image, passphrase, CryptSetup.LuksType.LUKS2); + } + @Override public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { return null; @@ -1544,6 +1789,13 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt */ KVMStoragePool srcPool = disk.getPool(); + if (ArrayUtils.isNotEmpty(srcPassphrase) && ENCRYPTED_COPY_POOL_TYPES.contains(srcPool.getType()) + && ENCRYPTED_COPY_POOL_TYPES.contains(destPool.getType())) { + // the volume keeps its passphrase when it is copied to another pool + return copyEncryptedPhysicalDisk(disk, name, destPool, timeout, srcPassphrase, + ArrayUtils.isNotEmpty(dstPassphrase) ? dstPassphrase : srcPassphrase); + } + /* Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, to support snapshots(backuped) as qcow2 files. */ PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? @@ -1628,19 +1880,28 @@ to support snapshots(backuped) as qcow2 files. */ * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling) */ logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); + String rbdDestPath = destPool.getSourceDir() + "/" + name; + // Only an image created by this copy may be removed again when the copy fails + boolean destCreatedByCopy = false; try { srcFile = new QemuImgFile(sourcePath, sourceFormat); - String rbdDestPath = destPool.getSourceDir() + "/" + name; String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath); destFile = new QemuImgFile(rbdDestFile, destFormat); + if (rbdImageExists(destPool, name)) { + // qemu-img can't convert into an existing image, and it can't be told if it is complete, so don't touch it + logger.error("RBD image {} already exists, it will not be overwritten by the copy of {}", rbdDestPath, srcFile.getFileName()); + return null; + } + destCreatedByCopy = true; + logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); qemu.convert(srcFile, destFile); logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); /* We have to stat the RBD image to see how big it became afterwards */ Rados r = new Rados(destPool.getAuthUserName()); - r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + r.confSet("mon_host", RbdUtils.getRbdMonitors(destPool.getSourceHost(), destPool.getSourcePort())); r.confSet("key", destPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); r.connect(); @@ -1659,15 +1920,23 @@ to support snapshots(backuped) as qcow2 files. */ r.ioCtxDestroy(io); } catch (QemuImgException | LibvirtException e) { String srcFilename = srcFile != null ? srcFile.getFileName() : null; - String destFilename = destFile != null ? destFile.getFileName() : null; - logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage())); + logger.error("Failed to convert from {} to RBD image {} the error was: {}", srcFilename, rbdDestPath, maskRbdKey(e.getMessage())); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } catch (RadosException e) { logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } catch (RbdException e) { logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } } else { /** @@ -1698,6 +1967,99 @@ to support snapshots(backuped) as qcow2 files. */ return newDisk; } + /** + * Copies an encrypted volume to another pool. The data is read through the encryption of the source and written + * through the encryption of the destination, which is created up front: an RBD image gets a native librbd LUKS2 + * header, a volume on a file based pool is a qemu LUKS encrypted QCOW2 file. Copying the encrypted data as is + * would not do: the destination may need another kind of encryption than the source, and an encrypted RBD clone + * reads the data it inherits from its (plaintext) parent unencrypted. Any backing chain of the source (an RBD + * parent or a QCOW2 backing file) is flattened into the destination. + */ + private KVMPhysicalDisk copyEncryptedPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] dstPassphrase) { + KVMStoragePool srcPool = disk.getPool(); + boolean srcOnRbd = srcPool.getType() == StoragePoolType.RBD; + boolean destOnRbd = destPool.getType() == StoragePoolType.RBD; + // the size of the disk as seen by the Instance: the LUKS2 header of an RBD image is carved out of the image + long guestSize = srcOnRbd ? disk.getVirtualSize() - LUKS2_HEADER_RESERVE_BYTES : disk.getVirtualSize(); + logger.debug("Copying encrypted volume {} from pool {} to {} in pool {}, with a size of {}", disk.getPath(), srcPool.getUuid(), + name, destPool.getUuid(), toHumanReadableSize(guestSize)); + + KVMPhysicalDisk newDisk = destPool.createPhysicalDisk(name, destOnRbd ? PhysicalDiskFormat.RAW : PhysicalDiskFormat.QCOW2, + Storage.ProvisioningType.THIN, destOnRbd ? guestSize + LUKS2_HEADER_RESERVE_BYTES : guestSize, dstPassphrase); + + List cephAuthFiles = new ArrayList<>(); + try (KeyFile srcKeyFile = new KeyFile(srcPassphrase); KeyFile destKeyFile = new KeyFile(dstPassphrase)) { + QemuImageOptions srcImageOpts = encryptedImageOptions(srcPool, disk.getName(), disk.getPath(), + disk.getFormat() == PhysicalDiskFormat.QCOW2 ? PhysicalDiskFormat.QCOW2 : PhysicalDiskFormat.LUKS, SRC_LUKS_SECRET_ID, cephAuthFiles); + srcImageOpts.setImageOptsFlag(true); + QemuImageOptions destImageOpts = encryptedImageOptions(destPool, name, newDisk.getPath(), PhysicalDiskFormat.QCOW2, + DEST_LUKS_SECRET_ID, cephAuthFiles); + + QemuImgFile srcFile = new QemuImgFile(disk.getPath(), srcOnRbd ? PhysicalDiskFormat.RAW : disk.getFormat()); + List secrets = List.of(createSecretObject(SRC_LUKS_SECRET_ID, srcKeyFile), createSecretObject(DEST_LUKS_SECRET_ID, destKeyFile)); + new QemuImg(timeout).convert(srcFile, null, null, null, secrets, srcImageOpts, destImageOpts, null, false, false, false, false, null, null); + } catch (IOException | QemuImgException | LibvirtException e) { + logger.error("Failed to copy encrypted volume {} to {} in pool {}: {}", disk.getPath(), name, destPool.getUuid(), e.getMessage()); + try { + destPool.deletePhysicalDisk(name, destOnRbd ? Storage.ImageFormat.RAW : Storage.ImageFormat.QCOW2); + } catch (Exception ex) { + logger.warn("Failed to remove {} from pool {} after the failed copy: {}", name, destPool.getUuid(), ex.getMessage()); + } + throw new CloudRuntimeException(String.format("Failed to copy encrypted volume %s to %s", disk.getPath(), name), e); + } finally { + for (Path file : cephAuthFiles) { + try { + Files.deleteIfExists(file); + } catch (IOException e) { + logger.warn("Failed to remove temporary file {}: {}", file, e.getMessage()); + } + } + } + + newDisk.setFormat(destOnRbd ? PhysicalDiskFormat.RAW : PhysicalDiskFormat.QCOW2); + newDisk.setVirtualSize(guestSize); + newDisk.setQemuEncryptFormat(destOnRbd ? QemuObject.EncryptFormat.LUKS2 : QemuObject.EncryptFormat.LUKS); + return newDisk; + } + + /** + * qemu image options addressing an encrypted volume: an RBD image (with librbd LUKS2 encryption, authenticated through + * a temporary ceph.conf and keyring that are added to {@code cephAuthFiles}) or a file of the given format. + */ + private QemuImageOptions encryptedImageOptions(KVMStoragePool pool, String rbdImage, String filePath, PhysicalDiskFormat fileFormat, + String secretId, List cephAuthFiles) throws IOException { + if (pool.getType() != StoragePoolType.RBD) { + return new QemuImageOptions(fileFormat, filePath, secretId); + } + Path keyring = Files.createTempFile("cs-ceph-", ".keyring", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------"))); + cephAuthFiles.add(keyring); + Files.writeString(keyring, "[client." + pool.getAuthUserName() + "]\n\tkey = " + pool.getAuthSecret() + "\n"); + Path conf = Files.createTempFile("cs-ceph-", ".conf", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------"))); + cephAuthFiles.add(conf); + String monHost = RbdUtils.getRbdMonitors(pool.getSourceHost(), pool.getSourcePort()); + Files.writeString(conf, "[global]\nmon_host = " + monHost + "\nkeyring = " + keyring + "\n"); + + Map options = new HashMap<>(); + options.put("driver", "rbd"); + options.put("pool", pool.getSourceDir()); + options.put("image", rbdImage); + options.put("conf", conf.toString()); + if (pool.getAuthUserName() != null) { + options.put("user", pool.getAuthUserName()); + } + options.put("encrypt.format", CryptSetup.LuksType.LUKS2.toString()); + options.put("encrypt.key-secret", secretId); + return new QemuImageOptions(options); + } + + private QemuObject createSecretObject(String secretId, KeyFile keyFile) { + Map params = new EnumMap<>(QemuObject.ObjectParameter.class); + params.put(QemuObject.ObjectParameter.ID, secretId); + params.put(QemuObject.ObjectParameter.FILE, keyFile.toString()); + return new QemuObject(QemuObject.ObjectType.SECRET, params); + } + @Override public boolean refresh(KVMStoragePool pool) { LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; @@ -1721,7 +2083,19 @@ private void refreshPool(StoragePool pool) throws LibvirtException { } private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { - vol.delete(0); + /** + * RBD volumes can have snapshots, and libvirt refuses to remove a volume while + * they exist. VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS tells the RBD storage backend + * to unprotect and remove any snapshots before removing the volume itself. + * + * libvirt-java has no named constant for this flag (added upstream in libvirt 1.2.20, + * commit 3c7590e0a4), so it's passed as a raw flag value here. + */ + int flags = 0; + if (pool.getType() == StoragePoolType.RBD) { + flags |= VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS; + } + vol.delete(flags); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index a8c32baa6ef3..e1a7a482db2b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -17,10 +17,14 @@ package com.cloud.hypervisor.kvm.storage; import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.List; import java.util.Map; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joda.time.Duration; @@ -35,11 +39,15 @@ import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.RbdUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; public class LibvirtStoragePool implements KVMStoragePool { + /** The environment variable which gives the Ceph key to the RBD heartbeat and VM-activity scripts. */ + protected static final String RBD_KEY_ENVIRONMENT_VARIABLE = "CEPH_KEY"; + protected Logger logger = LogManager.getLogger(getClass()); protected String uuid; protected long capacity; @@ -55,6 +63,7 @@ public class LibvirtStoragePool implements KVMStoragePool { protected StoragePool _pool; protected String authUsername; protected String authSecret; + protected String authSecretUUID; protected String sourceHost; protected int sourcePort; protected String sourceDir; @@ -282,6 +291,15 @@ public void setType(StoragePoolType type) { this.type = type; } + @Override + public String getAuthSecretUUID() { + return authSecretUUID; + } + + public void setAuthSecretUUID(String authSecretUUID) { + this.authSecretUUID = authSecretUUID; + } + public StoragePool getPool() { return this._pool; } @@ -330,37 +348,105 @@ public boolean isPoolSupportHA() { public String getHearthBeatPath() { if (StoragePoolType.NetworkFilesystem.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmheartbeat.sh"); } else if (StoragePoolType.SharedMountPoint.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmsmpheartbeat.sh"); + } else if (StoragePoolType.RBD.equals(type)) { + return findKvmHaScript("kvmheartbeat_rbd.py"); } return null; } + private String findKvmHaScript(String scriptName) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, scriptName); + if (scriptPath == null) { + throw new CloudRuntimeException(String.format("Unable to find script '%s' in directory: %s", scriptName, kvmScriptsDir)); + } + return scriptPath; + } + + /** + * Returns the environment of a heartbeat/VM-activity check script: for a RBD storage pool with cephx, the + * Ceph key, which is not an argument of the script as it would be visible in the process list. + */ + protected Map getHeartBeatScriptEnvironment() { + if (StoragePoolType.RBD.equals(type) && StringUtils.isNotBlank(authSecret)) { + return Map.of(RBD_KEY_ENVIRONMENT_VARIABLE, authSecret); + } + return null; + } + + /** + * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, the user; the key is in + * {@link #getHeartBeatScriptEnvironment()}) to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key" + * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}). + */ + protected void addRbdConnectionArgs(Script cmd) { + boolean hasUser = StringUtils.isNotBlank(authUsername); + boolean hasSecret = StringUtils.isNotBlank(authSecret); + if (hasUser != hasSecret) { + throw new CloudRuntimeException(String.format("The Ceph user and key must be either both set or both unset for the storage pool %s", uuid)); + } + cmd.add("-s", RbdUtils.getRbdMonitors(sourceHost, sourcePort)); + cmd.add("-o", sourceDir); + if (hasUser) { + cmd.add("-n", authUsername); + } + } + + /** + * Adds the arguments identifying the storage to a heartbeat/VM-activity check {@link Script}: + * the Ceph connection details for a RBD pool, or the NFS server, path and mount point otherwise. + */ + private void addPoolConnectionArgs(Script cmd, HAStoragePool pool) { + if (StoragePoolType.RBD.equals(type)) { + addRbdConnectionArgs(cmd); + } else { + cmd.add("-i", pool.getPoolIp()); + cmd.add("-p", pool.getPoolMountSourcePath()); + cmd.add("-m", pool.getMountDestPath()); + } + } - public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + /** + * Adds the volumes of the host to a VM-activity check {@link Script}. For a RBD storage pool they are given in + * a file, as they can be too many for an argument of a command (the limit is 128 KiB, about 3500 volumes), + * which would also be written to the log, with all the volumes, at each check. Otherwise, or if the file can not + * be written, they are an argument. + * + * @return the file, which has to be deleted after the script ran, or null if there is none + */ + protected File addVolumeListArgs(Script cmd, String volumes) { + if (StoragePoolType.RBD.equals(type)) { + File volumeListFile = null; + try { + volumeListFile = File.createTempFile("kvmha-volumes-", ".txt"); + Files.write(volumeListFile.toPath(), volumes.getBytes(StandardCharsets.UTF_8)); + cmd.add("-U", volumeListFile.getAbsolutePath()); + return volumeListFile; + } catch (IOException e) { + logger.warn("Unable to write the list of volumes to a file, giving it to the script as an argument: {}", e.getMessage()); + if (volumeListFile != null && !volumeListFile.delete()) { + logger.debug("Unable to delete the file {}", volumeListFile); + } + } + } + cmd.add("-u", volumes); + return null; + } + + public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String heartBeatHostId, boolean hostValidation) { Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); - cmd.add("-i", primaryStoragePool.getPoolIp()); - cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); - cmd.add("-m", primaryStoragePool.getMountDestPath()); + addPoolConnectionArgs(cmd, primaryStoragePool); if (hostValidation) { - cmd.add("-h", hostPrivateIp); + cmd.add("-h", heartBeatHostId); } else { cmd.add("-c"); } - return cmd.execute(); + return cmd.execute(new OutputInterpreter.OutputLogger(logger), getHeartBeatScriptEnvironment()); } @Override @@ -373,54 +459,71 @@ public String getStorageNodeId() { return null; } + /** + * The heartbeat of a host on an RBD pool is named by the GUID of the host, which stays the same if the host + * gets a new IP address. On the other pool types it is named by the private IP address of the host. + */ + protected String getHeartBeatHostId(HostTO host) { + if (!StoragePoolType.RBD.equals(type)) { + return host.getPrivateNetwork().getIp(); + } + if (StringUtils.isBlank(host.getGuid())) { + throw new CloudRuntimeException(String.format("The host %s has no GUID, which names its heartbeat on the RBD storage pool %s", host.getPrivateNetwork().getIp(), uuid)); + } + return host.getGuid(); + } + @Override public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { - String hostIp = host.getPrivateNetwork().getIp(); + String heartBeatHostId = getHeartBeatHostId(host); Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", hostIp); + addPoolConnectionArgs(cmd, pool); + cmd.add("-h", heartBeatHostId); cmd.add("-r"); cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); + String result = cmd.execute(parser, getHeartBeatScriptEnvironment()); String parsedLine = parser.getLine(); - logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); + logger.debug("Checking heart beat for host {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", heartBeatHostId, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + logger.warn("Checking heart beat for host {} with KVMHAChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", heartBeatHostId, cmd.toString(), parsedLine); return false; } else { - logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString()); + logger.debug("Checking heart beat for host {} with KVMHAChecker command [{}] succeeded.", heartBeatHostId, cmd.toString()); return true; } } @Override public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { - String hostIp = host.getPrivateNetwork().getIp(); + String heartBeatHostId = getHeartBeatHostId(host); Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", hostIp); - cmd.add("-u", volumeUUIDListString); + addPoolConnectionArgs(cmd, pool); + cmd.add("-h", heartBeatHostId); + File volumeListFile = addVolumeListArgs(cmd, volumeUUIDListString); cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); cmd.add("-d", String.valueOf(duration)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); + String result; + try { + result = cmd.execute(parser, getHeartBeatScriptEnvironment()); + } finally { + if (volumeListFile != null && !volumeListFile.delete()) { + logger.debug("Unable to delete the file {}", volumeListFile); + } + } String parsedLine = parser.getLine(); - logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); + logger.debug("Checking VM activity for host {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", heartBeatHostId, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + logger.warn("Checking VM activity for host {} with KVMHAVMActivityChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", heartBeatHostId, cmd.toString(), parsedLine); return false; } else { - logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString()); + logger.debug("Checking VM activity for host {} with KVMHAVMActivityChecker command [{}] succeeded.", heartBeatHostId, cmd.toString()); return true; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java index 4a577ef3400c..10e39cb5ffcb 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java @@ -86,8 +86,17 @@ public String[] toCommandFlag() { return new String[] { params.get(FILENAME_PARAM_KEY) }; } } + return toCommandFlag(QemuImg.IMAGE_OPTS_FLAG); + } + + /** + * Converts QemuImageOptions into the command strings under the given qemu-img flag, + * e.g. {@link QemuImg#TARGET_IMAGE_OPTS_FLAG} for a convert destination. + * @return array of strings representing the flag and its options value + */ + public String[] toCommandFlag(String flagName) { Map sorted = new TreeMap<>(params); String paramString = Joiner.on(",").withKeyValueSeparator("=").join(sorted); - return new String[] {"--image-opts", paramString}; + return new String[] {flagName, paramString}; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index cae6832999eb..49f531ed7c11 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -52,6 +52,8 @@ public class QemuImg { public static final String ENCRYPT_FORMAT = "encrypt.format"; public static final String ENCRYPT_KEY_SECRET = "encrypt.key-secret"; public static final String TARGET_ZERO_FLAG = "--target-is-zero"; + public static final String IMAGE_OPTS_FLAG = "--image-opts"; + public static final String TARGET_IMAGE_OPTS_FLAG = "--target-image-opts"; public static final String PREALLOCATION = "preallocation"; public static final long QEMU_2_10 = 2010000; public static final long QEMU_5_1 = 5001000; @@ -402,6 +404,21 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, convert(srcFile, destFile, null, options, qemuObjects, srcImageOpts, snapshotName, forceSourceFormat, false, false, false, null, null); } + /** + * Converts an image into an existing destination that is described by explicit image options + * ({@code --target-image-opts}) instead of a plain filename - for example an RBD image written + * through librbd encryption ({@code driver=rbd,...,encrypt.format=...}). The destination is + * never created ({@code -n} is implied) and must already exist with the wanted size and format. + * + * @param destImageOpts + * image options describing the existing destination; passed as --target-image-opts. + */ + public void convertIntoExistingTarget(final QemuImgFile srcFile, final Map options, + final List qemuObjects, final QemuImageOptions srcImageOpts, final QemuImageOptions destImageOpts, + final boolean forceSourceFormat) throws QemuImgException { + convert(srcFile, null, null, options, qemuObjects, srcImageOpts, destImageOpts, null, forceSourceFormat, false, false, false, null, null); + } + protected Map getResizeOptionsFromConvertOptions(final Map options) { if (MapUtils.isEmpty(options)) { return null; @@ -450,6 +467,25 @@ protected Map getResizeOptionsFromConvertOptions(final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat, boolean keepBitmaps, boolean outOfOrderWrites, boolean compress, Integer coroutines, Integer rateLimit) throws QemuImgException { + convert(srcFile, destFile, backingFile, options, qemuObjects, srcImageOpts, null, snapshotName, forceSourceFormat, keepBitmaps, outOfOrderWrites, compress, coroutines, + rateLimit); + } + + /** + * Converts an image from source to destination, optionally into an existing destination described by explicit image options + * ({@code --target-image-opts}) instead of a plain filename; see {@link #convertIntoExistingTarget}. All other parameters + * behave as documented above. + * + * @param destImageOpts + * If not null, the destination is described by these image options and {@code destFile} is unused. + */ + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuImgFile backingFile, final Map options, final List qemuObjects, + final QemuImageOptions srcImageOpts, final QemuImageOptions destImageOpts, final String snapshotName, final boolean forceSourceFormat, boolean keepBitmaps, + boolean outOfOrderWrites, boolean compress, Integer coroutines, Integer rateLimit) throws QemuImgException { + if (destImageOpts != null && this.version < QEMU_2_10) { + throw new QemuImgException(String.format("qemu >= 2.10 is required to convert into a destination described by %s", TARGET_IMAGE_OPTS_FLAG)); + } + Script script = new Script(_qemuImgPath, timeout); if (StringUtils.isNotBlank(snapshotName)) { String qemuPath = Script.runSimpleBashScript(getQemuImgPathScript); @@ -458,7 +494,10 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("convert"); - if (skipZero && Files.exists(Paths.get(destFile.getFileName()))) { + if (destImageOpts != null) { + // a destination described by image options always exists already; qemu-img requires -n with --target-image-opts + script.add("-n"); + } else if (skipZero && Files.exists(Paths.get(destFile.getFileName()))) { script.add("-n"); script.add(TARGET_ZERO_FLAG); script.add("-W"); @@ -469,8 +508,10 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("-n"); } - script.add("-O"); - script.add(destFile.getFormat().toString()); + if (destImageOpts == null) { + script.add("-O"); + script.add(destFile.getFormat().toString()); + } addBackingFileToConvertCommand(script, backingFile); addScriptOptionsFromMap(options, script); @@ -524,14 +565,19 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("--bitmaps"); } - script.add(destFile.getFileName()); + if (destImageOpts != null) { + script.add(destImageOpts.toCommandFlag(TARGET_IMAGE_OPTS_FLAG)); + } else { + script.add(destFile.getFileName()); + } final String result = script.execute(); if (result != null) { throw new QemuImgException(result); } - if (srcFile.getSize() < destFile.getSize()) { + // an image-options destination already exists with its final size; 'qemu-img resize' cannot address it by filename + if (destImageOpts == null && srcFile.getSize() < destFile.getSize()) { this.resize(destFile, destFile.getSize(), getResizeOptionsFromConvertOptions(options)); } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java index efeee04cb90f..511e91074969 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java @@ -54,6 +54,7 @@ public enum ObjectParameter { */ public enum EncryptFormat { LUKS("luks"), + LUKS2("luks2"), AES("aes"); private final String format; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java new file mode 100644 index 000000000000..e932cc889742 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.rbd; + +import com.cloud.utils.RbdUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.libvirt.LibvirtException; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Thin wrapper around the {@code rbd} CLI to apply native librbd LUKS encryption to an + * RBD image via {@code rbd encryption format}. This is only used at volume create time; + * runtime decryption is handled by libvirt/qemu through {@code }. + * + * The CLI dependency is intentionally isolated in this class so it can later be replaced + * by a native librbd (JNA) binding without touching callers. rados-java (0.x) does not + * expose the rbd_encryption_format API, hence the CLI for now. + * + * The command builders ({@code build*Script}) are separated from execution so the generated + * argv can be unit-tested without a live Ceph cluster (see {@code RbdEncryptionTest}). + */ +public class RbdEncryption { + protected Logger logger = LogManager.getLogger(getClass()); + + protected String commandPath = "rbd"; + + /** qemu secret id used to hand the LUKS passphrase to qemu-img during template import */ + protected static final String LUKS_SECRET_ID = "luks0"; + + public RbdEncryption() {} + + public RbdEncryption(String commandPath) { + this.commandPath = commandPath; + } + + private static String monSpec(String monHost, int monPort) { + return RbdUtils.getRbdMonitors(monHost, monPort); + } + + /** + * Apply a LUKS header to an existing RBD image so librbd can transparently encrypt it. + *

+ * cephx authentication is supplied via {@code --id} plus a temporary keyfile so the secret + * never appears on the command line. The LUKS passphrase is supplied via a temporary file + * ({@link KeyFile}). Both temp files are deleted when this method returns. + * + * @param monHost ceph monitor host + * @param monPort ceph monitor port (0 to omit) + * @param authUser cephx user (e.g. "cloudstack"); null to skip --id + * @param authSecret cephx secret/key (as used for ceph "key" config); null to skip --keyfile + * @param cephPool ceph pool name + * @param image rbd image name + * @param passphrase LUKS passphrase + * @param luksType LUKS1/LUKS2 (librbd engine supports both; LUKS2 recommended) + */ + public void format(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, byte[] passphrase, CryptSetup.LuksType luksType) { + final String imageSpec = cephPool + "/" + image; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot LUKS-format RBD image " + imageSpec + ": empty passphrase"); + } + try (KeyFile passFile = new KeyFile(passphrase); + KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { + final Script script = buildFormatScript(imageSpec, luksType, passFile.toString(), + monSpec(monHost, monPort), authUser, cephKeyFile.isSet() ? cephKeyFile.toString() : null); + final String result = script.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s: %s", luksType, imageSpec, result)); + } + logger.debug("Applied {} encryption to RBD image {}", luksType, imageSpec); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s", luksType, imageSpec), ex); + } + } + + protected Script buildFormatScript(String imageSpec, CryptSetup.LuksType luksType, String passFilePath, + String monSpec, String authUser, String cephKeyFilePath) { + final Script script = new Script(commandPath); + script.add("encryption"); + script.add("format"); + script.add(imageSpec); + script.add(luksType.toString()); + script.add(passFilePath); + script.add("--mon-host"); + script.add(monSpec); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFilePath != null) { + script.add("--keyfile"); + script.add(cephKeyFilePath); + } + return script; + } + + /** + * Resize an encrypted RBD image. librbd needs the passphrase so it can resize the encrypted + * payload (not just the raw image) and keep the LUKS header consistent. {@code newSizeBytes} is + * the usable (decrypted) size requested; rbd {@code --size} is expressed in MiB. + * + * @param allowShrink pass --allow-shrink when shrinking is permitted + */ + public void resize(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, long newSizeBytes, boolean allowShrink, byte[] passphrase) { + final String imageSpec = cephPool + "/" + image; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot resize encrypted RBD image " + imageSpec + ": empty passphrase"); + } + // rbd --size is in MiB; round up so a non-MiB-aligned request never shrinks the volume below what was asked for. + final long sizeMiB = (newSizeBytes + (1024L * 1024L) - 1) / (1024L * 1024L); + try (KeyFile passFile = new KeyFile(passphrase); + KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { + final Script script = buildResizeScript(imageSpec, sizeMiB, passFile.toString(), allowShrink, + monSpec(monHost, monPort), authUser, cephKeyFile.isSet() ? cephKeyFile.toString() : null); + final String result = script.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s to %d MiB: %s", imageSpec, sizeMiB, result)); + } + logger.debug("Resized encrypted RBD image {} to {} MiB", imageSpec, sizeMiB); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s", imageSpec), ex); + } + } + + protected Script buildResizeScript(String imageSpec, long sizeMiB, String passFilePath, boolean allowShrink, + String monSpec, String authUser, String cephKeyFilePath) { + final Script script = new Script(commandPath); + script.add("resize"); + script.add("--size"); + script.add(String.valueOf(sizeMiB)); + script.add(imageSpec); + script.add("--encryption-passphrase-file"); + script.add(passFilePath); + if (allowShrink) { + script.add("--allow-shrink"); + } + script.add("--mon-host"); + script.add(monSpec); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFilePath != null) { + script.add("--keyfile"); + script.add(cephKeyFilePath); + } + return script; + } + + /** + * Import a template into an already-created, already-LUKS-formatted RBD image by writing it + * THROUGH the librbd encryption layer with {@code qemu-img convert -n} (so the data lands + * encrypted). This is how encrypted root disks are populated: we never clone-then-format a + * plaintext template (that leaves the inherited OS data unreadable) — instead we format an + * empty image and convert the template into it. + * + * Exactly one source must be given: an RBD image ({@code srcRbdPool}+{@code srcRbdImage}) or a + * local file ({@code srcFilePath}[+{@code srcFileFormat}]). cephx auth is provided to qemu-img + * via a temporary ceph.conf + keyring (deleted on return). The conversion itself goes through + * {@link QemuImg#convertIntoExistingTarget}. + */ + public void importTemplate(String srcRbdPool, String srcRbdImage, + String srcFilePath, String srcFileFormat, + String monHost, int monPort, String authUser, String authSecret, + String cephPool, String destImage, byte[] passphrase, CryptSetup.LuksType luksType) { + final String imageSpec = cephPool + "/" + destImage; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot import template into encrypted RBD image " + imageSpec + ": empty passphrase"); + } + Path conf = null; + Path keyring = null; + try (KeyFile passFile = new KeyFile(passphrase)) { + // These temp files hold the cephx secret; create them 0600 up front (matching KeyFile) rather than relying on the umask. + final FileAttribute ownerOnly = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")); + keyring = Files.createTempFile("cs-ceph-", ".keyring", ownerOnly); + Files.writeString(keyring, "[client." + authUser + "]\n\tkey = " + authSecret + "\n"); + conf = Files.createTempFile("cs-ceph-", ".conf", ownerOnly); + Files.writeString(conf, "[global]\nmon_host = " + monSpec(monHost, monPort) + "\nkeyring = " + keyring + "\n"); + + QemuImgFile srcQemuFile; + QemuImageOptions srcImageOpts; + boolean forceSourceFormat = false; + if (srcRbdImage != null) { + srcQemuFile = new QemuImgFile(srcRbdPool + "/" + srcRbdImage, QemuImg.PhysicalDiskFormat.RAW); + srcImageOpts = new QemuImageOptions(rbdImageOptions(srcRbdPool, srcRbdImage, conf.toString(), authUser)); + srcImageOpts.setImageOptsFlag(true); + } else { + QemuImg.PhysicalDiskFormat srcFormat = srcFileFormat != null ? QemuImg.PhysicalDiskFormat.valueOf(srcFileFormat.toUpperCase()) : null; + srcQemuFile = srcFormat != null ? new QemuImgFile(srcFilePath, srcFormat) : new QemuImgFile(srcFilePath); + srcImageOpts = new QemuImageOptions(srcFilePath); + if (srcFormat != null) { + // emit the source as --image-opts driver=,file.filename= (the -f equivalent) + srcImageOpts.setImageOptsFlag(true); + forceSourceFormat = true; + } + } + + Map destParams = rbdImageOptions(cephPool, destImage, conf.toString(), authUser); + destParams.put("encrypt.format", luksType.toString()); + destParams.put("encrypt.key-secret", LUKS_SECRET_ID); + QemuImageOptions destImageOpts = new QemuImageOptions(destParams); + + EnumMap secretParams = new EnumMap<>(QemuObject.ObjectParameter.class); + secretParams.put(QemuObject.ObjectParameter.ID, LUKS_SECRET_ID); + secretParams.put(QemuObject.ObjectParameter.FILE, passFile.toString()); + QemuObject luksSecret = new QemuObject(QemuObject.ObjectType.SECRET, secretParams); + + QemuImg qemu = createQemuImg(); + qemu.convertIntoExistingTarget(srcQemuFile, null, List.of(luksSecret), srcImageOpts, destImageOpts, forceSourceFormat); + logger.debug("Imported template into encrypted RBD image {}", imageSpec); + } catch (IOException | QemuImgException | LibvirtException ex) { + throw new CloudRuntimeException(String.format("Failed to import template into encrypted RBD image %s", imageSpec), ex); + } finally { + deleteQuietly(conf); + deleteQuietly(keyring); + } + } + + /** + * Seam for unit tests; {@link QemuImg} probes the qemu version through libvirt on construction. + */ + protected QemuImg createQemuImg() throws QemuImgException, LibvirtException { + return new QemuImg(0); + } + + /** qemu image options addressing an RBD image (as used with --image-opts / --target-image-opts). */ + private static Map rbdImageOptions(String cephPool, String image, String confPath, String authUser) { + Map opts = new HashMap<>(); + opts.put("driver", "rbd"); + opts.put("pool", cephPool); + opts.put("image", image); + opts.put("conf", confPath); + if (authUser != null) { + opts.put("user", authUser); + } + return opts; + } + + private static void deleteQuietly(Path p) { + if (p == null) { + return; + } + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort cleanup of the temporary ceph auth files + } + } + + /** + * Best-effort probe that the local rbd CLI supports the encryption subcommand. + */ + public boolean isSupported() { + final Script script = new Script(commandPath); + script.add("help"); + script.add("encryption"); + script.add("format"); + return script.execute() == null; + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java new file mode 100644 index 000000000000..50e7427aabd6 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.joda.time.Duration; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; +import com.cloud.agent.api.to.HostTO; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase; +import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest { + + private static final String POOL_UUID = "7684f6e5-fd91-3435-b04e-59ae0efb3914"; + private static final String RBD_SCRIPT = "/scripts/kvmvmactivity_rbd.py"; + private static final String SCRIPT = "/scripts/kvmvmactivity.sh"; + + private final LibvirtCheckVMActivityOnStoragePoolCommandWrapper wrapper = new LibvirtCheckVMActivityOnStoragePoolCommandWrapper(); + private final LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + private final KVMStoragePool pool = mock(KVMStoragePool.class); + private final CheckVMActivityOnStoragePoolCommand command = mock(CheckVMActivityOnStoragePoolCommand.class); + + private void setUp(StoragePoolType type, boolean supportsHa) { + StorageFilerTO filer = mock(StorageFilerTO.class); + when(filer.getType()).thenReturn(type); + when(filer.getUuid()).thenReturn(POOL_UUID); + when(command.getPool()).thenReturn(filer); + KVMStoragePoolManager poolManager = mock(KVMStoragePoolManager.class); + when(poolManager.getStoragePool(type, POOL_UUID)).thenReturn(pool); + when(resource.getStoragePoolMgr()).thenReturn(poolManager); + when(pool.isPoolSupportHA()).thenReturn(supportsHa); + } + + private KVMHABase.HAStoragePool addHaPool() { + KVMHABase.HAStoragePool haPool = new KVMHABase.HAStoragePool(pool, "10.0.0.1", "cloudstack", KVMHABase.PoolType.PrimaryStorage); + KVMHAMonitor monitor = mock(KVMHAMonitor.class); + when(monitor.getStoragePool(POOL_UUID)).thenReturn(haPool); + when(resource.getMonitor()).thenReturn(monitor); + return haPool; + } + + private void expectActivityCheck(String volumes, long suspectTime, boolean activity) { + when(command.getHost()).thenReturn(mock(HostTO.class)); + when(command.getVolumeList()).thenReturn(volumes); + when(command.getSuspectTimeInSeconds()).thenReturn(suspectTime); + when(pool.hasVmActivity(any(), any(), any(), anyString(), anyString(), anyLong())).thenReturn(activity); + } + + @Test + public void testRbdPoolIsCheckedWithTheScriptOfRbd() { + setUp(StoragePoolType.RBD, true); + KVMHABase.HAStoragePool haPool = addHaPool(); + expectActivityCheck("volume-1,volume-2", 123L, false); + + try (MockedStatic