Skip to content

add ha disable to vmKill #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TheHeroBrine422 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

add ha disable to vmKill #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4769a2d
add ha disable to vmKill
TheHeroBrine422 Jul 5, 2026
62c731b
full ha disabled solution
TheHeroBrine422 Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion apps/dashboard/src/lib/server/backends/proxmox/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import type {
PveNextId,
PveCreateQemuParams,
PveClusterResource,
PveAgentNetworkInterface
PveAgentNetworkInterface,
PveHaResource
} from './types';

export interface ProxmoxClientConfig {
Expand Down Expand Up @@ -417,4 +418,19 @@ export class ProxmoxClient {
.json<PveResponse<{ result: PveAgentNetworkInterface[] }>>();
return res.data.result;
}

async updateHAResources(sid: string, params: Record<string, string>): Promise<void> {
await this.api
.put(`cluster/ha/resources/${sid}`, {
body: this.toForm(params)
})
.json<PveResponse<null>>();
}

async getHaResource(sid: string): Promise<PveHaResource> {
const res = await this.api
.get(`cluster/ha/resources/${sid}`)
.json<PveResponse<PveHaResource>>();
return res.data;
}
}
164 changes: 137 additions & 27 deletions apps/dashboard/src/lib/server/backends/proxmox/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -664,31 +664,84 @@ export class ProxmoxBackend implements VmBackend {
await this.client.waitForTask(node, upid);
}

private async ensureVmStopped(node: string, vmid: number): Promise<void> {
const status = await this.client.getQemuVm(node, vmid);
if (status.status === 'stopped') return;
// Poll until the VM reaches the given power state. Used to observe the
// result of fire-and-forget HA operations, which return no task to wait on.
private async waitForVmState(
node: string,
vmid: number,
target: 'stopped' | 'running',
timeoutMs = 60_000
): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if ((await this.client.getQemuVm(node, vmid)).status === target) return;
await new Promise((r) => setTimeout(r, 1_000));
}
throw new Error(
`VM ${vmid} on node ${node} did not reach ${target} state within ${timeoutMs}ms`
);
}

let stopUpid: string;
try {
stopUpid = await this.client.stopVm(node, vmid, { overruleShutdown: true });
} catch (err) {
if (!(err instanceof HTTPError) || err.response.status !== 400) throw err;
stopUpid = await this.client.stopVm(node, vmid);
// A VM stuck in an HA error state can't be stopped normally — the stop routes
// through the HA manager and fails. Disabling HA breaks that state and stops
// the VM. updateHAResources is fire-and-forget (no UPID), so we poll for the
// VM to actually stop.
private async disableHaAndWaitStopped(node: string, vmid: number): Promise<void> {
await this.client.updateHAResources(`vm:${vmid}`, { state: 'disabled' });
await this.waitForVmState(node, vmid, 'stopped');
}

// Re-arm HA management at the given state and wait for the config to confirm.
// updateHAResources is fire-and-forget, so we poll the resource config. A
// timeout is non-fatal: the VM's power state is already correct, HA just
// hasn't caught up yet.
private async setHaStateAndWait(vmid: number, state: 'stopped' | 'started'): Promise<void> {
await this.client.updateHAResources(`vm:${vmid}`, { state });
const deadline = Date.now() + 30_000;
while (Date.now() < deadline) {
try {
const current = (await this.client.getHaResource(`vm:${vmid}`)).state;
// 'enabled' is the documented alias of 'started'
if (current === state || (state === 'started' && current === 'enabled')) return;
} catch {
// config may briefly be unreadable during the transition
}
await new Promise((r) => setTimeout(r, 1_000));
}
console.warn(`HA state for vm:${vmid} did not confirm ${state} within 30s`);
}

private async ensureVmStopped(node: string, vmid: number): Promise<void> {
if ((await this.client.getQemuVm(node, vmid)).status === 'stopped') return;

try {
let stopUpid: string;
try {
stopUpid = await this.client.stopVm(node, vmid, { overruleShutdown: true });
} catch (err) {
if (!(err instanceof HTTPError) || err.response.status !== 400) throw err;
stopUpid = await this.client.stopVm(node, vmid);
}
await this.client.waitForTask(node, stopUpid);
} catch (err) {
const current = await this.client.getQemuVm(node, vmid);
if (current.status !== 'stopped') throw err;
} catch (stopErr) {
if ((await this.client.getQemuVm(node, vmid)).status === 'stopped') return;
// Stuck HA state: disable HA to stop the VM. No need to re-arm HA —
// the VM is being deleted.
console.warn(
`ensureVmStopped: stop of VM ${vmid} on node ${node} failed, trying HA-disable fallback:`,
stopErr
);
try {
await this.disableHaAndWaitStopped(node, vmid);
} catch (haErr) {
throw new Error(
`stopping VM ${vmid} on node ${node} failed: ${stopErr}; HA-disable fallback also failed: ${haErr}`
);
}
}

const deadline = Date.now() + 60_000;
while (Date.now() < deadline) {
const current = await this.client.getQemuVm(node, vmid);
if (current.status === 'stopped') return;
await new Promise((r) => setTimeout(r, 1_000));
}
throw new Error(`VM ${vmid} on node ${node} did not reach stopped state within 60s`);
// Safety net: confirm the VM is actually stopped before deleting.
await this.waitForVmState(node, vmid, 'stopped');
}

private async destroyVm(node: string, vmid: number): Promise<string | undefined> {
Expand All @@ -710,28 +763,85 @@ export class ProxmoxBackend implements VmBackend {
async startVm(id: string, proxmoxId?: number): Promise<void> {
clearProxmoxReadCaches();
const { node, vmid } = await this.resolve(id, proxmoxId);
const upid = await this.client.startVm(node, vmid);
await this.client.waitForTask(node, upid);
try {
const upid = await this.client.startVm(node, vmid);
await this.client.waitForTask(node, upid);
return;
} catch (err) {
if ((await this.client.getQemuVm(node, vmid)).status === 'running') return;
// Stuck VM: break the HA state (stops it), start it fresh, re-arm HA.
console.warn(
`startVm: start of VM ${vmid} on node ${node} failed, trying HA-disable fallback:`,
err
);
await this.disableHaAndWaitStopped(node, vmid);
const upid = await this.client.startVm(node, vmid);
await this.client.waitForTask(node, upid);
await this.setHaStateAndWait(vmid, 'started');
}
}

async stopVm(id: string, proxmoxId?: number): Promise<void> {
clearProxmoxReadCaches();
const { node, vmid } = await this.resolve(id, proxmoxId);
const upid = await this.client.shutdownVm(node, vmid);
await this.client.waitForTask(node, upid);
try {
const upid = await this.client.shutdownVm(node, vmid);
await this.client.waitForTask(node, upid);
return;
} catch (err) {
if ((await this.client.getQemuVm(node, vmid)).status === 'stopped') return;
// Stuck VM: the graceful shutdown routes through the HA manager and
// fails, so disable HA to stop it, then re-arm HA as stopped.
console.warn(
`stopVm: shutdown of VM ${vmid} on node ${node} failed, trying HA-disable fallback:`,
err
);
await this.disableHaAndWaitStopped(node, vmid);
await this.setHaStateAndWait(vmid, 'stopped');
}
}

async killVm(id: string, proxmoxId?: number): Promise<void> {
clearProxmoxReadCaches();
const { node, vmid } = await this.resolve(id, proxmoxId);
const upid = await this.client.stopVm(node, vmid);
await this.client.waitForTask(node, upid);
try {
const upid = await this.client.stopVm(node, vmid);
await this.client.waitForTask(node, upid);
return;
} catch (err) {
if ((await this.client.getQemuVm(node, vmid)).status === 'stopped') return;
// Stuck VM: the hard stop routes through the HA manager and fails, so
// disable HA to stop it, then re-arm HA as stopped.
console.warn(
`killVm: stop of VM ${vmid} on node ${node} failed, trying HA-disable fallback:`,
err
);
await this.disableHaAndWaitStopped(node, vmid);
await this.setHaStateAndWait(vmid, 'stopped');
}
}

async rebootVm(id: string, proxmoxId?: number): Promise<void> {
clearProxmoxReadCaches();
const { node, vmid } = await this.resolve(id, proxmoxId);
const upid = await this.client.rebootVm(node, vmid);
await this.client.waitForTask(node, upid);
try {
const upid = await this.client.rebootVm(node, vmid);
await this.client.waitForTask(node, upid);
return;
} catch (err) {
if ((await this.client.getQemuVm(node, vmid)).status === 'running') return;
// Stuck VM: a normal reboot routes through the HA manager and fails.
// Break the HA state (stops the VM), start it fresh, re-arm HA.
// If the VM is down, this degrades to a start — the reboot's target
// state is 'running', which the start achieves.
console.warn(
`rebootVm: reboot of VM ${vmid} on node ${node} failed, trying HA-disable fallback:`,
err
);
await this.disableHaAndWaitStopped(node, vmid);
const upid = await this.client.startVm(node, vmid);
await this.client.waitForTask(node, upid);
await this.setHaStateAndWait(vmid, 'started');
}
}
}
14 changes: 14 additions & 0 deletions apps/dashboard/src/lib/server/backends/proxmox/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,20 @@ export interface PveClusterResource {
uptime?: number;
}

export interface PveHaResource {
sid: string;
state: 'started' | 'stopped' | 'enabled' | 'disabled' | 'ignored';
type?: string;
group?: string;
comment?: string;
digest?: string;
max_relocate?: number;
max_restart?: number;
'auto-rebalance'?: boolean;
failback?: boolean;
[key: string]: unknown;
}

export interface PveStorageContent {
volid: string;
content: string;
Expand Down