Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,25 @@ private void deployClient(final KVMHostConnectedContext context, List<ExternalPr
extPs.getUuid(), extPs.getName(), context.getInventory().getUuid(), context.getInventory().getName()));

// one deploy pass redeploys the baseline client plus every exposed
// protocol's data path (e.g. the vhost SPDK target) on reconnect
List<String> protocols = Q.New(PrimaryStorageOutputProtocolRefVO.class)
// protocol's data path (e.g. the vhost SPDK target). a protocol already
// Connected on this host is skipped so reconnect doesn't bounce a live target.
List<String> protocols = new ArrayList<>(Q.New(PrimaryStorageOutputProtocolRefVO.class)
.eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, extPs.getUuid())
.select(PrimaryStorageOutputProtocolRefVO_.outputProtocol)
.listValues());

List<String> connected = Q.New(ExternalPrimaryStorageHostProtocolRefVO.class)
.eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, extPs.getUuid())
.eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, context.getInventory().getUuid())
.eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected)
.select(ExternalPrimaryStorageHostProtocolRefVO_.protocol)
.listValues();
protocols.removeAll(connected);

if (protocols.isEmpty()) {
compl.done();
return;
}

extPsFactory.getNodeSvc(extPs.getUuid()).deployClient(context.getInventory(), protocols, new Completion(compl) {
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import org.zstack.utils.gson.JSONObjectUtil
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference

/**
Expand Down Expand Up @@ -174,6 +175,7 @@ class ZbsVhostVolumeCase extends SubCase {
testVhostVmStartActivationChain()
testAddProtocolPreparesHostsAndRecordsProtocolRefs()
testProtocolRecoveryBackoff()
testReconnectSkipsConnectedProtocolDeploy()
}
}

Expand Down Expand Up @@ -817,6 +819,64 @@ class ZbsVhostVolumeCase extends SubCase {
}
}

// reconnect re-runs the host-connect prepare flow, but a protocol already Connected
// on this host must NOT be redeployed: bouncing a live vhost SPDK target on every
// reconnect would interrupt running vhost VMs. only down/absent protocols deploy.
void testReconnectSkipsConnectedProtocolDeploy() {
HostInventory host = env.inventoryByName("kvm-1") as HostInventory
AtomicInteger deployCount = new AtomicInteger(0)

env.simulator(ZbsStorageController.DEPLOY_VHOST_PATH) { HttpEntity<String> e, EnvSpec spec ->
deployCount.incrementAndGet()
return new ZbsStorageController.AgentResponse()
}
env.simulator(ZbsStorageController.PREPARE_VHOST_TARGET_ENV_PATH) { HttpEntity<String> e, EnvSpec spec ->
return new ZbsStorageController.AgentResponse()
}
env.simulator(ZbsStorageController.VHOST_TARGET_HEALTH_PATH) { HttpEntity<String> e, EnvSpec spec ->
def rsp = new ZbsStorageController.VhostTargetHealthRsp()
rsp.targetRunning = true
return rsp
}
env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity<String> e ->
def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd)
if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) {
def vrsp = new ZbsStorageController.CreateVolumeRsp()
vrsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}".toString()
return vrsp
}
return rsp
}

attachPrimaryStorageToCluster {
primaryStorageUuid = ps.uuid
clusterUuid = cluster.uuid
}

// wait until the vhost protocol row is Connected on this host
retryInSecs {
assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class)
.eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid)
.eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, host.uuid)
.eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString())
.eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected)
.isExists()
}

// every exposed protocol Connected -> reconnect must not deploy any of them
deployCount.set(0)
reconnectHost { uuid = host.uuid }
Thread.sleep(2000)
assert deployCount.get() == 0 : \
"reconnect redeployed an already-Connected vhost target ${deployCount.get()} time(s); " +
"host-connect must skip protocols already Connected"

detachPrimaryStorageFromCluster {
primaryStorageUuid = ps.uuid
clusterUuid = cluster.uuid
}
}

private MessageReply syncSend(org.zstack.header.message.Message msg) {
AtomicReference<MessageReply> ref = new AtomicReference<>()
CountDownLatch done = new CountDownLatch(1)
Expand Down