diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index fa8a7a79aa37..a25f038e9620 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -2566,7 +2566,10 @@ protected void verifyLiveMigrationForKVM(Map volumeDataSt } boolean isSrcAndDestPoolPowerFlexStorage = srcStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && destStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex); - if (srcStoragePoolVO.isManaged() && !isSrcAndDestPoolPowerFlexStorage && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) { + boolean isSrcAndDestPoolFiberChannelStorage = srcStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.FiberChannel) && destStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.FiberChannel); + boolean fiberChannelVmOnline = isSrcAndDestPoolFiberChannelStorage && volumeInfo.getAttachedVM() != null && volumeInfo.getAttachedVM().getState() == VirtualMachine.State.Running; + + if (srcStoragePoolVO.isManaged() && !isSrcAndDestPoolPowerFlexStorage && !fiberChannelVmOnline && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) { throw new CloudRuntimeException("Migrating a volume online with KVM from managed storage is not currently supported."); } @@ -2766,6 +2769,27 @@ private Map getVolumeDetails(VolumeInfo volumeInfo) { return volumeDetails; } + private boolean shouldAttemptLiveFiberChannelMigration(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { + if (srcVolumeInfo == null || destVolumeInfo == null) { + return false; + } + + StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId()); + StoragePoolVO destPool = _storagePoolDao.findById(destVolumeInfo.getPoolId()); + + if (srcPool == null || destPool == null) { + return false; + } + + if (srcPool.getPoolType() != StoragePoolType.FiberChannel || destPool.getPoolType() != StoragePoolType.FiberChannel) { + return false; + } + + VirtualMachine attachedVm = srcVolumeInfo.getAttachedVM(); + + return attachedVm != null && attachedVm.getState() == VirtualMachine.State.Running; + } + private Map getSnapshotDetails(SnapshotInfo snapshotInfo) { Map snapshotDetails = new HashMap<>(); @@ -3021,8 +3045,11 @@ private String migrateVolumeForKVM(VolumeInfo srcVolumeInfo, VolumeInfo destVolu _volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); + boolean fiberChannelOnline = shouldAttemptLiveFiberChannelMigration(srcVolumeInfo, destVolumeInfo); + int waitTimeout = fiberChannelOnline ? StorageManager.KvmStorageOnlineMigrationWait.value() : StorageManager.KvmStorageOfflineMigrationWait.value(); + MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcVolumeInfo.getTO(), destVolumeInfo.getTO(), - srcDetails, destDetails, StorageManager.KvmStorageOfflineMigrationWait.value()); + srcDetails, destDetails, waitTimeout); _volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); handleQualityOfServiceForVolumeMigration(destVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 808c319b40f2..a2bb7daabae9 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -124,6 +124,8 @@ public class KvmNonManagedStorageSystemDataMotionTest { Host host1; @Mock Host host2; + @Mock + com.cloud.vm.VirtualMachine attachedVm; Map migrationMap; @@ -478,6 +480,23 @@ public void testVerifyLiveMigrationMapForKVM() { kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap); } + @Test + public void testVerifyLiveMigrationMapForKVMManagedFiberChannelAllowed() { + lenient().when(pool1.isManaged()).thenReturn(true); + lenient().when(pool2.isManaged()).thenReturn(true); + lenient().when(pool1.getPoolType()).thenReturn(Storage.StoragePoolType.FiberChannel); + lenient().when(pool2.getPoolType()).thenReturn(Storage.StoragePoolType.FiberChannel); + lenient().when(pool1.getId()).thenReturn(POOL_1_ID); + lenient().when(pool2.getId()).thenReturn(POOL_2_ID); + lenient().when(volumeInfo1.getAttachedVM()).thenReturn(attachedVm); + when(attachedVm.getState()).thenReturn(com.cloud.vm.VirtualMachine.State.Running); + + Map fiberChannelMigrationMap = new HashMap<>(); + fiberChannelMigrationMap.put(volumeInfo1, dataStore2); + + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(fiberChannelMigrationMap); + } + @Test(expected = CloudRuntimeException.class) public void testVerifyLiveMigrationMapForKVMNotExistingSource() { when(primaryDataStoreDao.findById(POOL_1_ID)).thenReturn(null); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java index 609a1225118a..422aaf4320d1 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java @@ -60,6 +60,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.DeleteVMSnapshotAnswer; @@ -94,6 +95,10 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.cloud.vm.snapshot.dao.VMSnapshotDetailsDao; +import com.cloud.event.dao.UsageEventDao; +import com.cloud.event.UsageEventUtils; +import com.cloud.user.dao.AccountDao; +import com.cloud.dc.dao.DataCenterDao; import junit.framework.TestCase; @@ -144,10 +149,33 @@ public class VMSnapshotStrategyKVMTest extends TestCase{ @Inject VMSnapshotDetailsDao vmSnapshotDetailsDao; + private UsageEventDao usageEventDao; + private AccountDao accountDao; + private DataCenterDao dataCenterDao; + @Override @Before public void setUp() throws Exception { ComponentContext.initComponentsLifeCycle(); + initialiseUsageEventUtils(); + } + + private void initialiseUsageEventUtils() { + usageEventDao = Mockito.mock(UsageEventDao.class); + accountDao = Mockito.mock(AccountDao.class); + dataCenterDao = Mockito.mock(DataCenterDao.class); + ConfigurationDao configDao = Mockito.mock(ConfigurationDao.class); + + UsageEventUtils usageEventUtils = new UsageEventUtils(); + ReflectionTestUtils.setField(usageEventUtils, "usageEventDao", usageEventDao); + ReflectionTestUtils.setField(usageEventUtils, "accountDao", accountDao); + ReflectionTestUtils.setField(usageEventUtils, "dcDao", dataCenterDao); + ReflectionTestUtils.setField(usageEventUtils, "configDao", configDao); + ReflectionTestUtils.invokeMethod(usageEventUtils, "init"); + + Mockito.lenient().when(usageEventDao.persist(Mockito.any())).thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.lenient().doNothing().when(usageEventDao).saveDetails(Mockito.anyLong(), Mockito.anyMap()); + Mockito.lenient().when(configDao.getValue("publish.usage.events")).thenReturn("false"); } @Test diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index bf002b37f355..74f3fea44520 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -1132,6 +1132,10 @@ public void setSerial(String serial) { this._serial = serial; } + public String getSerial() { + return _serial; + } + public void setLibvirtDiskEncryptDetails(LibvirtDiskEncryptDetails details) { this.encryptDetails = details; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index b79735830cf6..b16e94d3f25c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -25,6 +25,7 @@ import com.cloud.agent.api.storage.MigrateVolumeCommand; import com.cloud.agent.api.to.DiskTO; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; import com.cloud.hypervisor.kvm.resource.disconnecthook.VolumeMigrationCancelHook; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; @@ -37,6 +38,7 @@ import java.io.File; import java.io.IOException; import java.io.StringWriter; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -74,11 +76,15 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper details) { + if (details != null && details.get(DiskTO.IQN) != null) { + return details.get(DiskTO.IQN); + } + return volumeObjectTO != null ? volumeObjectTO.getPath() : null; + } + + private DiskDef locateSourceDiskDefinition(List disks, KVMPhysicalDisk srcPhysicalDisk, Map srcDetails) { + String expectedPath = srcPhysicalDisk != null ? srcPhysicalDisk.getPath() : null; + String expectedSerial = srcDetails != null ? srcDetails.get(DiskTO.SCSI_NAA_DEVICE_ID) : null; + + if (StringUtils.isNotBlank(expectedPath)) { + for (DiskDef disk : disks) { + if (StringUtils.isNotBlank(disk.getDiskPath()) && pathsReferToSameDevice(expectedPath, disk.getDiskPath())) { + return disk; + } + } + } + + if (StringUtils.isNotBlank(expectedSerial)) { + for (DiskDef disk : disks) { + if (expectedSerial.equalsIgnoreCase(disk.getSerial())) { + return disk; + } + } + } + + return null; + } + + private boolean pathsReferToSameDevice(String expectedPath, String candidatePath) { + if (StringUtils.equals(expectedPath, candidatePath)) { + return true; + } + String expectedToken = extractLastToken(expectedPath); + String candidateToken = extractLastToken(candidatePath); + return StringUtils.isNotBlank(expectedToken) && StringUtils.equalsIgnoreCase(expectedToken, candidateToken); + } + + private String extractLastToken(String path) { + if (StringUtils.isBlank(path)) { + return null; + } + int idx = path.lastIndexOf('/'); + return idx >= 0 ? path.substring(idx + 1) : path; + } + + private String buildFiberChannelDestinationDiskXml(DiskDef sourceDiskDef, String destDevicePath) { + StringBuilder diskXml = new StringBuilder(); + diskXml.append(""); + diskXml.append(""); + diskXml.append(""); + diskXml.append(""); + diskXml.append(""); + return diskXml.toString(); + } + protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { // Source Details @@ -183,6 +247,152 @@ protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand } } + protected boolean shouldAttemptFiberChannelLiveMigration(MigrateVolumeCommand command, PrimaryDataStoreTO srcPrimaryDataStore, PrimaryDataStoreTO destPrimaryDataStore) { + if (srcPrimaryDataStore == null || destPrimaryDataStore == null) { + return false; + } + + if (!Storage.StoragePoolType.FiberChannel.equals(srcPrimaryDataStore.getPoolType()) || + !Storage.StoragePoolType.FiberChannel.equals(destPrimaryDataStore.getPoolType())) { + return false; + } + + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + + return srcVolumeObjectTO != null && StringUtils.isNotBlank(srcVolumeObjectTO.getVmName()); + } + + protected MigrateVolumeAnswer migrateFiberChannelVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + if (srcVolumeObjectTO == null || destVolumeObjectTO == null || StringUtils.isBlank(srcVolumeObjectTO.getVmName())) { + return migrateRegularVolume(command, libvirtComputingResource); + } + + String vmName = srcVolumeObjectTO.getVmName(); + Map srcDetails = command.getSrcDetails(); + Map destDetails = command.getDestDetails(); + + PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + PrimaryDataStoreTO destPrimaryDataStore = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + + KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); + KVMStoragePool sourceStoragePool = null; + KVMStoragePool destStoragePool = null; + Domain dm = null; + VolumeMigrationCancelHook cancelHook = null; + boolean migrationSucceeded = false; + + String srcIdentifier = resolveVolumeIdentifier(srcVolumeObjectTO, srcDetails); + String destIdentifier = resolveVolumeIdentifier(destVolumeObjectTO, destDetails); + + if (StringUtils.isBlank(srcIdentifier) || StringUtils.isBlank(destIdentifier)) { + return migrateRegularVolume(command, libvirtComputingResource); + } + + try { + sourceStoragePool = storagePoolManager.getStoragePool(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid()); + if (!sourceStoragePool.connectPhysicalDisk(srcIdentifier, srcDetails)) { + return new MigrateVolumeAnswer(command, false, "Unable to connect source Fibre Channel volume on hypervisor", srcIdentifier); + } + KVMPhysicalDisk srcPhysicalDisk = storagePoolManager.getPhysicalDisk(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid(), srcIdentifier); + if (srcPhysicalDisk == null) { + return new MigrateVolumeAnswer(command, false, "Unable to obtain source Fibre Channel disk handle", srcIdentifier); + } + + destStoragePool = storagePoolManager.getStoragePool(destPrimaryDataStore.getPoolType(), destPrimaryDataStore.getUuid()); + if (!destStoragePool.connectPhysicalDisk(destIdentifier, destDetails)) { + return new MigrateVolumeAnswer(command, false, "Unable to connect destination Fibre Channel volume on hypervisor", destIdentifier); + } + + if (destVolumeObjectTO.getPath() == null) { + destVolumeObjectTO.setPath(destIdentifier); + } + + KVMPhysicalDisk destPhysicalDisk = storagePoolManager.getPhysicalDisk(destPrimaryDataStore.getPoolType(), destPrimaryDataStore.getUuid(), destIdentifier); + if (destPhysicalDisk == null) { + return new MigrateVolumeAnswer(command, false, "Unable to obtain destination Fibre Channel disk handle", destIdentifier); + } + String destDevicePath = destPhysicalDisk.getPath(); + + LibvirtUtilitiesHelper helper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = helper.getConnection(); + dm = libvirtComputingResource.getDomain(conn, vmName); + if (dm == null) { + return new MigrateVolumeAnswer(command, false, "Unable to locate libvirt domain for VM " + vmName, null); + } + + DomainInfo.DomainState domainState = dm.getInfo().state; + if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return migrateRegularVolume(command, libvirtComputingResource); + } + + List disks = libvirtComputingResource.getDisks(conn, vmName); + DiskDef srcDiskDef = locateSourceDiskDefinition(disks, srcPhysicalDisk, srcDetails); + if (srcDiskDef == null) { + return new MigrateVolumeAnswer(command, false, "Unable to match Fibre Channel disk within VM definition", null); + } + + String diskLabel = srcDiskDef.getDiskLabel(); + String destinationDiskXml = buildFiberChannelDestinationDiskXml(srcDiskDef, destDevicePath); + + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); + TypedParameter[] parameters = new TypedParameter[] { parameter }; + + cancelHook = new VolumeMigrationCancelHook(dm, diskLabel); + libvirtComputingResource.addDisconnectHook(cancelHook); + + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.PROCESSING_IN_BACKEND); + + dm.blockCopy(diskLabel, destinationDiskXml, parameters, Domain.BlockCopyFlags.REUSE_EXT); + + MigrateVolumeAnswer answer = checkBlockJobStatus(command, dm, diskLabel, srcPhysicalDisk.getPath(), destDevicePath, libvirtComputingResource, conn, null); + migrationSucceeded = answer != null && answer.getResult(); + + if (answer != null) { + libvirtComputingResource.createOrUpdateLogFileForCommand(command, migrationSucceeded ? Command.State.COMPLETED : Command.State.FAILED); + return answer; + } + + migrationSucceeded = true; + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.COMPLETED); + return new MigrateVolumeAnswer(command, true, null, destIdentifier); + } catch (LibvirtException e) { + logger.warn("Fibre Channel live volume migration failed due to libvirt error", e); + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.FAILED); + return new MigrateVolumeAnswer(command, false, "Libvirt error during Fibre Channel migration: " + e.getMessage(), null); + } catch (Exception e) { + logger.warn("Fibre Channel live volume migration failed", e); + libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.FAILED); + return new MigrateVolumeAnswer(command, false, "Fibre Channel migration failed: " + e.getMessage(), null); + } finally { + if (cancelHook != null) { + libvirtComputingResource.removeDisconnectHook(cancelHook); + } + if (dm != null) { + try { + dm.free(); + } catch (LibvirtException e) { + logger.trace("Ignoring libvirt error while freeing domain", e); + } + } + if (!migrationSucceeded && destStoragePool != null) { + try { + destStoragePool.disconnectPhysicalDisk(destIdentifier); + } catch (Exception e) { + logger.warn("Unable to disconnect destination Fibre Channel disk after failed migration", e); + } + } + if (sourceStoragePool != null) { + try { + sourceStoragePool.disconnectPhysicalDisk(srcIdentifier); + } catch (Exception e) { + logger.warn("Unable to disconnect source Fibre Channel disk after migration", e); + } + } + } + } + protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found int waitTimeInSec = command.getWait();