diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index a5815821f6a..76f3f0df970 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -202,11 +202,25 @@ private void deployClient(final KVMHostConnectedContext context, List protocols = Q.New(PrimaryStorageOutputProtocolRefVO.class) + // protocol's data path (e.g. the vhost SPDK target). a protocol already + // Connected on this host is skipped so reconnect doesn't bounce a live target. + List protocols = new ArrayList<>(Q.New(PrimaryStorageOutputProtocolRefVO.class) .eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, extPs.getUuid()) .select(PrimaryStorageOutputProtocolRefVO_.outputProtocol) + .listValues()); + + List connected = Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, extPs.getUuid()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, context.getInventory().getUuid()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .select(ExternalPrimaryStorageHostProtocolRefVO_.protocol) .listValues(); + protocols.removeAll(connected); + + if (protocols.isEmpty()) { + compl.done(); + return; + } extPsFactory.getNodeSvc(extPs.getUuid()).deployClient(context.getInventory(), protocols, new Completion(compl) { @Override diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy index 1e881620db9..9e17288744f 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy @@ -41,6 +41,7 @@ import org.zstack.utils.gson.JSONObjectUtil import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicReference /** @@ -174,6 +175,7 @@ class ZbsVhostVolumeCase extends SubCase { testVhostVmStartActivationChain() testAddProtocolPreparesHostsAndRecordsProtocolRefs() testProtocolRecoveryBackoff() + testReconnectSkipsConnectedProtocolDeploy() } } @@ -817,6 +819,64 @@ class ZbsVhostVolumeCase extends SubCase { } } + // reconnect re-runs the host-connect prepare flow, but a protocol already Connected + // on this host must NOT be redeployed: bouncing a live vhost SPDK target on every + // reconnect would interrupt running vhost VMs. only down/absent protocols deploy. + void testReconnectSkipsConnectedProtocolDeploy() { + HostInventory host = env.inventoryByName("kvm-1") as HostInventory + AtomicInteger deployCount = new AtomicInteger(0) + + env.simulator(ZbsStorageController.DEPLOY_VHOST_PATH) { HttpEntity e, EnvSpec spec -> + deployCount.incrementAndGet() + return new ZbsStorageController.AgentResponse() + } + env.simulator(ZbsStorageController.PREPARE_VHOST_TARGET_ENV_PATH) { HttpEntity e, EnvSpec spec -> + return new ZbsStorageController.AgentResponse() + } + env.simulator(ZbsStorageController.VHOST_TARGET_HEALTH_PATH) { HttpEntity e, EnvSpec spec -> + def rsp = new ZbsStorageController.VhostTargetHealthRsp() + rsp.targetRunning = true + return rsp + } + env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity e -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd) + if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) { + def vrsp = new ZbsStorageController.CreateVolumeRsp() + vrsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}".toString() + return vrsp + } + return rsp + } + + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + + // wait until the vhost protocol row is Connected on this host + retryInSecs { + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() + } + + // every exposed protocol Connected -> reconnect must not deploy any of them + deployCount.set(0) + reconnectHost { uuid = host.uuid } + Thread.sleep(2000) + assert deployCount.get() == 0 : \ + "reconnect redeployed an already-Connected vhost target ${deployCount.get()} time(s); " + + "host-connect must skip protocols already Connected" + + detachPrimaryStorageFromCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + } + private MessageReply syncSend(org.zstack.header.message.Message msg) { AtomicReference ref = new AtomicReference<>() CountDownLatch done = new CountDownLatch(1)