Support VDDK imports into RBD storage

Author: andrijapanicsb

PendingReleaseNotes

@@ -39,3 +39,11 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+4.23.0.0:
+  * VMware-to-KVM import using VDDK can import powered-off VMware VMs directly
+    into Ceph RBD primary storage when forceconverttopool is enabled and the
+    selected KVM conversion host supports qemu RBD access and in-place virt-v2v
+    finalization. Hosts without in-place finalization support can still use the
+    staged VDDK flow, converting to temporary qcow2 storage before copying the
+    finalized disks into RBD.

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -38,6 +38,7 @@ public class RemoteInstanceTO implements Serializable {
     private String datacenterName;
     private String clusterName;
     private String hostName;
+    private String vmwareMoref;
 
     public RemoteInstanceTO() {
     }
@@ -100,4 +101,12 @@ public String getClusterName() {
     public String getHostName() {
         return hostName;
     }
+
+    public String getVmwareMoref() {
+        return vmwareMoref;
+    }
+
+    public void setVmwareMoref(String vmwareMoref) {
+        this.vmwareMoref = vmwareMoref;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/VmwareVddkSourceDiskTO.java

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.io.Serializable;
+
+public class VmwareVddkSourceDiskTO implements Serializable {
+
+    private String diskId;
+    private String sourceDiskPath;
+    private long capacityBytes;
+    private Integer position;
+
+    public VmwareVddkSourceDiskTO() {
+    }
+
+    public VmwareVddkSourceDiskTO(String diskId, String sourceDiskPath, long capacityBytes, Integer position) {
+        this.diskId = diskId;
+        this.sourceDiskPath = sourceDiskPath;
+        this.capacityBytes = capacityBytes;
+        this.position = position;
+    }
+
+    public String getDiskId() {
+        return diskId;
+    }
+
+    public String getSourceDiskPath() {
+        return sourceDiskPath;
+    }
+
+    public long getCapacityBytes() {
+        return capacityBytes;
+    }
+
+    public Integer getPosition() {
+        return position;
+    }
+}

api/src/main/java/com/cloud/host/Host.java

@@ -62,6 +62,10 @@ public static String[] toStrings(Host.Type... types) {
     String HOST_VDDK_VERSION = "host.vddk.version";
     String HOST_OVFTOOL_VERSION = "host.ovftool.version";
     String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
+    String HOST_VDDK_RBD_DIRECT_IMPORT_SUPPORT = "host.vddk.rbd.direct.import.support";
+    String HOST_VIRTV2V_INPLACE_SUPPORT = "host.virt.v2v.inplace.support";
+    String HOST_QEMU_IMG_RBD_SUPPORT = "host.qemu.img.rbd.support";
+    String HOST_RBD_QEMU_COPY_SUPPORT = "host.rbd.qemu.copy.support";
     String HOST_SSH_PORT = "host.ssh.port";
 
     int DEFAULT_SSH_PORT = 22;

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java

@@ -153,7 +153,8 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
     private Long importInstanceHostId;
 
     @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class,
-            description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.")
+            description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM. " +
+                    "When " + ApiConstants.USE_VDDK + " and " + ApiConstants.FORCE_CONVERT_TO_POOL + " are true, this can be an RBD primary storage pool for direct RBD import.")
     private Long convertStoragePoolId;
 
     @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN,
@@ -183,7 +184,9 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
             type = CommandType.BOOLEAN,
             since = "4.22.1",
             description = "(only for importing VMs from VMware to KVM) optional - if true, uses VDDK on the KVM conversion host for converting the VM. " +
-                    "This parameter is mutually exclusive with " + ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES + ".")
+                    "This parameter is mutually exclusive with " + ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES + ". " +
+                    "With " + ApiConstants.FORCE_CONVERT_TO_POOL + "=true and an RBD conversion pool, the source VMware VM must be powered off and " +
+                    "the conversion host must support qemu RBD access and in-place virt-v2v finalization.")
     private Boolean useVddk;
 
 

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -66,6 +66,7 @@ public enum PowerState {
 
     private String bootType;
     private String bootMode;
+    private String vmwareMoref;
 
     public String getName() {
         return name;
@@ -234,6 +235,14 @@ public void setBootMode(String bootMode) {
         this.bootMode = bootMode;
     }
 
+    public String getVmwareMoref() {
+        return vmwareMoref;
+    }
+
+    public void setVmwareMoref(String vmwareMoref) {
+        this.vmwareMoref = vmwareMoref;
+    }
+
     public static class Disk {
         private String diskId;
 

core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java

@@ -19,6 +19,7 @@
 public class CheckConvertInstanceCommand extends Command {
     boolean checkWindowsGuestConversionSupport = false;
     boolean useVddk = false;
+    boolean checkVddkRbdDirectImportSupport = false;
     String vddkLibDir;
 
     public CheckConvertInstanceCommand() {
@@ -50,6 +51,14 @@ public void setUseVddk(boolean useVddk) {
         this.useVddk = useVddk;
     }
 
+    public boolean isCheckVddkRbdDirectImportSupport() {
+        return checkVddkRbdDirectImportSupport;
+    }
+
+    public void setCheckVddkRbdDirectImportSupport(boolean checkVddkRbdDirectImportSupport) {
+        this.checkVddkRbdDirectImportSupport = checkVddkRbdDirectImportSupport;
+    }
+
     public String getVddkLibDir() {
         return vddkLibDir;
     }

core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java

@@ -18,8 +18,11 @@
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.RemoteInstanceTO;
+import com.cloud.agent.api.to.VmwareVddkSourceDiskTO;
 import com.cloud.hypervisor.Hypervisor;
 
+import java.util.List;
+
 public class ConvertInstanceCommand extends Command {
 
     private RemoteInstanceTO sourceInstance;
@@ -35,6 +38,7 @@ public class ConvertInstanceCommand extends Command {
     private String vddkLibDir;
     private String vddkTransports;
     private String vddkThumbprint;
+    private List<VmwareVddkSourceDiskTO> vmwareVddkSourceDisks;
 
     public ConvertInstanceCommand() {
     }
@@ -126,6 +130,14 @@ public void setVddkThumbprint(String vddkThumbprint) {
         this.vddkThumbprint = vddkThumbprint;
     }
 
+    public List<VmwareVddkSourceDiskTO> getVmwareVddkSourceDisks() {
+        return vmwareVddkSourceDisks;
+    }
+
+    public void setVmwareVddkSourceDisks(List<VmwareVddkSourceDiskTO> vmwareVddkSourceDisks) {
+        this.vmwareVddkSourceDisks = vmwareVddkSourceDisks;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;

core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java

@@ -18,13 +18,15 @@
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.RemoteInstanceTO;
+import com.cloud.storage.Storage;
 
 import java.util.List;
 
 public class ImportConvertedInstanceCommand extends Command {
 
     private RemoteInstanceTO sourceInstance;
     private List<String> destinationStoragePools;
+    private List<Storage.StoragePoolType> destinationStoragePoolTypes;
     private DataStoreTO conversionTemporaryLocation;
     private String temporaryConvertUuid;
     private boolean forceConvertToPool;
@@ -34,15 +36,24 @@ public ImportConvertedInstanceCommand() {
 
     public ImportConvertedInstanceCommand(RemoteInstanceTO sourceInstance,
                                           List<String> destinationStoragePools,
+                                          List<Storage.StoragePoolType> destinationStoragePoolTypes,
                                           DataStoreTO conversionTemporaryLocation, String temporaryConvertUuid,
                                           boolean forceConvertToPool) {
         this.sourceInstance = sourceInstance;
         this.destinationStoragePools = destinationStoragePools;
+        this.destinationStoragePoolTypes = destinationStoragePoolTypes;
         this.conversionTemporaryLocation = conversionTemporaryLocation;
         this.temporaryConvertUuid = temporaryConvertUuid;
         this.forceConvertToPool = forceConvertToPool;
     }
 
+    public ImportConvertedInstanceCommand(RemoteInstanceTO sourceInstance,
+                                          List<String> destinationStoragePools,
+                                          DataStoreTO conversionTemporaryLocation, String temporaryConvertUuid,
+                                          boolean forceConvertToPool) {
+        this(sourceInstance, destinationStoragePools, null, conversionTemporaryLocation, temporaryConvertUuid, forceConvertToPool);
+    }
+
     public RemoteInstanceTO getSourceInstance() {
         return sourceInstance;
     }
@@ -51,6 +62,10 @@ public List<String> getDestinationStoragePools() {
         return destinationStoragePools;
     }
 
+    public List<Storage.StoragePoolType> getDestinationStoragePoolTypes() {
+        return destinationStoragePoolTypes;
+    }
+
     public DataStoreTO getConversionTemporaryLocation() {
         return conversionTemporaryLocation;
     }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -808,8 +808,13 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi
                 String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT);
                 String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR);
                 String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION);
+                String virtv2vInPlaceSupport = detailsMap.get(Host.HOST_VIRTV2V_INPLACE_SUPPORT);
+                String qemuImgRbdSupport = detailsMap.get(Host.HOST_QEMU_IMG_RBD_SUPPORT);
+                String rbdQemuCopySupport = detailsMap.get(Host.HOST_RBD_QEMU_COPY_SUPPORT);
+                String vddkRbdDirectImportSupport = detailsMap.get(Host.HOST_VDDK_RBD_DIRECT_IMPORT_SUPPORT);
                 logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host);
-                if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) {
+                if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion,
+                        virtv2vInPlaceSupport, qemuImgRbdSupport, rbdQemuCopySupport, vddkRbdDirectImportSupport)) {
                     _hostDao.loadDetails(host);
                     boolean updateNeeded = false;
                     if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) {
@@ -844,6 +849,22 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi
                         }
                         updateNeeded = true;
                     }
+                    if (StringUtils.isNotBlank(virtv2vInPlaceSupport) && !virtv2vInPlaceSupport.equals(host.getDetails().get(Host.HOST_VIRTV2V_INPLACE_SUPPORT))) {
+                        host.getDetails().put(Host.HOST_VIRTV2V_INPLACE_SUPPORT, virtv2vInPlaceSupport);
+                        updateNeeded = true;
+                    }
+                    if (StringUtils.isNotBlank(qemuImgRbdSupport) && !qemuImgRbdSupport.equals(host.getDetails().get(Host.HOST_QEMU_IMG_RBD_SUPPORT))) {
+                        host.getDetails().put(Host.HOST_QEMU_IMG_RBD_SUPPORT, qemuImgRbdSupport);
+                        updateNeeded = true;
+                    }
+                    if (StringUtils.isNotBlank(rbdQemuCopySupport) && !rbdQemuCopySupport.equals(host.getDetails().get(Host.HOST_RBD_QEMU_COPY_SUPPORT))) {
+                        host.getDetails().put(Host.HOST_RBD_QEMU_COPY_SUPPORT, rbdQemuCopySupport);
+                        updateNeeded = true;
+                    }
+                    if (StringUtils.isNotBlank(vddkRbdDirectImportSupport) && !vddkRbdDirectImportSupport.equals(host.getDetails().get(Host.HOST_VDDK_RBD_DIRECT_IMPORT_SUPPORT))) {
+                        host.getDetails().put(Host.HOST_VDDK_RBD_DIRECT_IMPORT_SUPPORT, vddkRbdDirectImportSupport);
+                        updateNeeded = true;
+                    }
                     if (updateNeeded) {
                         _hostDao.saveDetails(host);
                     }