diff --git a/.github/workflows/vgpu-guest.yml b/.github/workflows/vgpu-guest.yml new file mode 100644 index 000000000..a853b1692 --- /dev/null +++ b/.github/workflows/vgpu-guest.yml @@ -0,0 +1,93 @@ +name: vGPU guest references + +on: + pull_request: + paths: + - 'static/examples/vgpu-guest/**' + - 'hack/test_vgpu_guest.py' + - 'hack/Test-VgpuGuest.Tests.ps1' + - 'content/en/docs/next/virtualization/vgpu-licensing.md' + - 'content/en/docs/next/virtualization/vgpu.md' + - 'content/en/docs/next/operations/configuration/platform-package.md' + - '.github/workflows/vgpu-guest.yml' + push: + branches: [main] + paths: + - 'static/examples/vgpu-guest/**' + - 'hack/test_vgpu_guest.py' + - 'hack/Test-VgpuGuest.Tests.ps1' + - 'content/en/docs/next/virtualization/vgpu-licensing.md' + - 'content/en/docs/next/virtualization/vgpu.md' + - 'content/en/docs/next/operations/configuration/platform-package.md' + - '.github/workflows/vgpu-guest.yml' + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + awk: [mawk, gawk] + env: + VGPU_TEST_AWK: ${{ matrix.awk }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Syntax and sandbox tests + run: | + shellcheck --version + python3 --version + sh -n static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.sh + shellcheck --shell=sh static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.sh + python3 -I hack/test_vgpu_guest.py + - name: Verify systemd dependency wiring + shell: bash + run: | + units=$(mktemp --directory) + cp static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.service "$units/" + mkdir "$units/nvidia-gridd.service.d" + cp static/examples/vgpu-guest/v1/linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf "$units/nvidia-gridd.service.d/" + printf '[Service]\nExecStart=/usr/bin/true\n' > "$units/nvidia-gridd.service" + printf '[Unit]\n' > "$units/sysinit.target" + printf '[Unit]\n' > "$units/basic.target" + printf '[Unit]\n' > "$units/shutdown.target" + # Only executable existence is substituted; the sandbox suite pins the installed path. + sed --in-place 's|ExecStart=/usr/local/sbin/cozystack-vgpu-license|ExecStart=/usr/bin/true|' "$units/cozystack-vgpu-license.service" + SYSTEMD_UNIT_PATH="$units" systemd-analyze verify "$units/cozystack-vgpu-license.service" "$units/nvidia-gridd.service" + - name: Check downloadable checksums + working-directory: static/examples/vgpu-guest/v1 + run: sha256sum --check SHA256SUMS + + windows: + runs-on: windows-2025 + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + shell: [powershell, pwsh] + defaults: + run: + shell: ${{ matrix.shell }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Test Windows token cleanup and service sequence + run: | + Write-Host "PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" + Import-Module Pester -RequiredVersion 5.9.0 -ErrorAction Stop + $config = New-PesterConfiguration + $config.Run.Path = './hack/Test-VgpuGuest.Tests.ps1' + $config.Run.Throw = $true + $config.Run.PassThru = $true + $config.TestResult.Enabled = $true + $config.TestResult.OutputPath = Join-Path $env:RUNNER_TEMP 'vgpu-guest.xml' + $result = Invoke-Pester -Configuration $config + if ($null -eq $result -or $result.TotalCount -eq 0 -or $result.PassedCount -eq 0) { + throw 'No vGPU guest tests passed' + } diff --git a/content/en/docs/next/operations/configuration/platform-package.md b/content/en/docs/next/operations/configuration/platform-package.md index 2b17aeb24..65b150801 100644 --- a/content/en/docs/next/operations/configuration/platform-package.md +++ b/content/en/docs/next/operations/configuration/platform-package.md @@ -227,6 +227,15 @@ registries: | `resources.memoryAllocationRatio` | `1` | Memory allocation ratio: `1/memoryAllocationRatio` memory requested per unit of configured memory. | | `resources.ephemeralStorageAllocationRatio` | `40` | Ephemeral storage allocation ratio: `1/ephemeralStorageAllocationRatio` ephemeral storage requested per unit of configured storage. | +#### GPU + +| Value | Default | Description | +| --- | --- | --- | +| `gpu.vgpu` | `false` | Grants the root tenant vGPU entitlement. Child tenants also need their own Tenant `gpu.vgpu` and an entitled parent at every level. This is separate from GPU quota. | +| `gpu.resourceNames` | `[]` | Additional GPU extended resource names requested only by pods, such as `nvidia.com/gpu`, beyond names discovered from GPUClass objects and KubeVirt's NVIDIA/mediated permitted-device entries. Used to recognize GPU requests and prevent a VM using class entries from also supplying its own raw GPU requests. It neither advertises devices nor grants entitlement. | + +These keys belong to the unreleased GPU class configuration. For licensed classes and guest image preparation, see [vGPU guest licensing]({{% ref "/docs/next/virtualization/vgpu-licensing" %}}). Keep the hoster's GPUClass catalog and the device plugin's actual resource names aligned; the extra resource-name list does not create catalog entries. + #### KubeVirt | Value | Default | Description | diff --git a/content/en/docs/next/virtualization/vgpu-licensing.md b/content/en/docs/next/virtualization/vgpu-licensing.md new file mode 100644 index 000000000..cc9993c17 --- /dev/null +++ b/content/en/docs/next/virtualization/vgpu-licensing.md @@ -0,0 +1,171 @@ +--- +title: "vGPU guest licensing" +description: "Prepare platform-licensed Linux and Windows guests, connect them to NVIDIA licensing, and plan rotation." +weight: 45 +--- + +This guide describes the unreleased GPU class license-delivery contract and reference guest assets. It does not establish that a particular guest image, GPU, driver or license server has passed hardware validation. Complete the [acceptance checklist](#guest-acceptance-checklist) for the versions you deploy. + +## Choose the licensing mode + +A hoster-licensed vGPU class has a `license`: the name of the hoster's source Secret and a `featureType` of `1` (default) or `4`. The GPU controller copies its token into entitled tenant namespaces. The VMI webhook attaches the class's copy as a disk labelled `GPULICENSE`. The disk contains `client_configuration_token.tok` and `gridd.conf`; it is not a boot disk. + +A BYOL class has no `license`. Its tenant configures licensing inside the guest and still needs vGPU entitlement. **Do not install the reference unit or startup task in a BYOL image.** Without the platform disk, Linux selects an empty managed token directory and Windows clears the default token folder. These references do not support switching one image between platform licensing and BYOL. + +The hoster's NVIDIA entitlement is the default for licensed classes. Before distributing an image containing an NVIDIA driver, check the applicable distribution terms and restrict access where those terms require it. These downloads contain no NVIDIA driver, proprietary binary or token. + +## Platform preparation + +1. Enable root entitlement with platform `gpu.vgpu: true`; it defaults to `false`. Grant `gpu.vgpu` on each child Tenant through its parent's administrator. Every ancestor must be entitled. The effective namespace label is `namespace.cozystack.io/vgpu: "true"`. Entitlement is separate from GPU quota. +2. Define the hoster's `GPUClass` catalog and source license Secrets. Use class entries in VMInstance or KubernetesNodes to select a licensed offer. Raw `name` entries remain the compatibility path; they do not opt into class-based licensing. +3. Configure platform `gpu.resourceNames` for GPU resource names in the deployment. See the [platform reference]({{% ref "/docs/next/operations/configuration/platform-package" %}}). +4. Confirm that the controller created `gpuclass--license` in the entitled namespace. Inspect names, keys and conditions, not Secret payloads. The class reports `LicenseSecretMissing` if its source Secret is missing or lacks the token key; existing copies are retained in that case. A zero-length token is not useful guest media. + +A class-based vGPU start without effective entitlement is refused. When a required license copy does not exist, the launcher waits in `ContainerCreating` with a `FailedMount` event naming the missing Secret and the VM shows `Starting`; it does not deliberately boot without that copy. Check the class condition and source Secret first. Neither a pending launcher nor the presence of the disk proves that the guest has obtained an NVIDIA license. + +The delivery contract is [gpudevice §6](https://github.com/cozystack/community/blob/026c4190c9c1d8d0e458e2d60bb621c2b4ecd6b9/design-proposals/gpudevice/README.md#6-vgpu-entitlement-and-license-delivery), with [decision 0018](https://github.com/cozystack/community/blob/026c4190c9c1d8d0e458e2d60bb621c2b4ecd6b9/design-proposals/gpudevice/decisions/0018-a-vgpu-license-reaches-each-vm-as-a-secret-volume.md). Guest responsibilities follow the accepted [vGPU VM licensing design](https://github.com/cozystack/community/blob/024d8b9133520a17e6610271bed7ad78b0b3edfa/design-proposals/vgpu-vms/README.md). + +## Download one reference revision + +| Asset | Download | +| --- | --- | +| Linux script | [cozystack-vgpu-license.sh](/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.sh) | +| Linux oneshot | [cozystack-vgpu-license.service](/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.service) | +| NVIDIA client drop-in | [cozystack-vgpu-license.conf](/examples/vgpu-guest/v1/linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf) | +| Windows script | [Set-CozystackVgpuLicense.ps1](/examples/vgpu-guest/v1/windows/Set-CozystackVgpuLicense.ps1) | +| Checksums | [SHA256SUMS](/examples/vgpu-guest/v1/SHA256SUMS) | + +`v1` names the reference contract, not a Cozystack release or an immutable file revision. For reproducible installation, select a reviewed **full website commit SHA** and fetch every file and its checksum manifest from that same commit. Review and authenticate the selected commit through your image-build process; downloading a checksum beside a file is not independent authentication. Do not pipe downloads into a shell. + +On the Linux image-build machine, in a new empty directory: + +```bash +REV='' +BASE="https://raw.githubusercontent.com/cozystack/website/${REV}/static/examples/vgpu-guest/v1" +for FILE in linux/cozystack-vgpu-license.sh linux/cozystack-vgpu-license.service linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf windows/Set-CozystackVgpuLicense.ps1 SHA256SUMS; do + mkdir --parents "$(dirname "$FILE")" + curl --fail --show-error --location --output "$FILE" "${BASE}/${FILE}" +done +sha256sum --check SHA256SUMS +``` + +For a Windows build, download the same revision's script and manifest before installation. Verify the Windows entry without executing the script: + +```powershell +$Revision = '' +$Base = "https://raw.githubusercontent.com/cozystack/website/$Revision/static/examples/vgpu-guest/v1" +Invoke-WebRequest -Uri "$Base/windows/Set-CozystackVgpuLicense.ps1" -OutFile './Set-CozystackVgpuLicense.ps1' +Invoke-WebRequest -Uri "$Base/SHA256SUMS" -OutFile './SHA256SUMS' +$Entry = @(Get-Content -LiteralPath './SHA256SUMS' | Where-Object { $_ -match '^[a-f0-9]{64} windows/Set-CozystackVgpuLicense.ps1$' }) +if ($Entry.Count -ne 1) { throw 'Missing or ambiguous checksum' } +$Expected = $Entry[0].Substring(0, 64) +if ((Get-FileHash -LiteralPath './Set-CozystackVgpuLicense.ps1' -Algorithm SHA256).Hash -ne $Expected) { throw 'Checksum mismatch' } +``` + +## Linux image installation + +Use a systemd-based image with the NVIDIA guest driver and its `nvidia-gridd.service` installed. Runtime prerequisites are POSIX `/bin/sh`, `id`, `find`, `mkdir`, `rmdir`, `rm`, `cp`, `mv`, `chmod`, `od` and `awk`; util-linux `lsblk`, `blkid`, `findmnt`, `mount` and `umount`; and systemd's `udevadm`. Python is an offline test dependency only. The script uses POSIX utility syntax where no portable long option exists and util-linux long options for block and mount operations; BusyBox compatibility is not claimed. Verify the selected executables and option support in the actual guest image, including `findmnt`'s `ID` and `MAJ:MIN` columns. + +`/run/cozystack-vgpu-license/token` and its private sibling `.stage` must share one `tmpfs` or `ramfs` mount. The script verifies the actual token filesystem and staging mount identity, not just `/run`'s usual backing. Configuration staging is private and on the same mount as `/etc/nvidia/gridd.conf`. The config directory must be writable at client startup, including on immutable images. Configure these paths in the image; the script does not install packages or create a memory mount. + +Execution is serialized by systemd, with root-owned directories that are not writable by other users. Do not invoke the script manually alongside the unit or another root config writer. Path checks and rename do not protect against concurrent root pathname or mount replacement. Token and config publication each use a completed same-filesystem rename; they are not a two-file transaction. The required service dependency keeps the client from reading intermediate state. + +Run these commands as root in the image build, from the verified download directory, with the client stopped: + +```bash +systemctl stop nvidia-gridd.service +install --mode=0755 linux/cozystack-vgpu-license.sh /usr/local/sbin/cozystack-vgpu-license +install --mode=0644 linux/cozystack-vgpu-license.service /etc/systemd/system/cozystack-vgpu-license.service +install --directory --mode=0755 /etc/systemd/system/nvidia-gridd.service.d +install --mode=0644 linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf /etc/systemd/system/nvidia-gridd.service.d/cozystack-vgpu-license.conf +systemctl daemon-reload +systemctl enable nvidia-gridd.service +``` + +The oneshot is pulled in by the client's `Requires=` and `After=` drop-in; it has no boot-target enablement. Do not add a `RequiredBy=` boot target. A failed preparation blocks **the licensing client, not VM boot**. Installing these files into an already-running licensed VM does not demonstrate first-boot ordering. Stop the client before installation and validate a fresh boot of the resulting image. + +At each client start or restart the script first removes old files only from its managed token directory, then sets `ClientConfigTokenPath=/run/cozystack-vgpu-license/token`. It waits up to 10 seconds for queued udev events with `udevadm settle`, then enumerates labels with `lsblk`. This requires a working udev database and complete block-device visibility in the guest. A successful inventory without a candidate or label link is the reference's no-media case: preparation succeeds with an empty directory and preserves `FeatureType`. Blank or encrypted data disks with other labels do not prevent this. A discovery timeout or command failure, stale label link, multiple candidates, or a failed fresh `blkid --probe` of a candidate is an explicit error, not no-media. In particular, probe exit status 2 does not prove absence. This is ordinary systemd/udev discovery, not an independent scan proving that every raw superblock lacks the label. + +With one confirmed `GPULICENSE`, the script mounts read-only with `nodev,nosuid,noexec` and verifies the exact mountpoint and source device identity. It validates a single `FeatureType=1` or `FeatureType=4` and copies only the nonempty regular, single-link token file. Media `gridd.conf` must be ASCII text with LF or CRLF records; duplicate active `FeatureType` assignments, even identical ones, fail. The parser never executes configuration, trusts a media-supplied token path, or interprets the token format: token bytes are copied unchanged, including binary bytes. Structural checks cannot establish NVIDIA token validity. + +The class-based VMI integration permits vGPUs from one class per VM and attaches one license disk, including when requesting multiple GPUs of that class. The guest reference rejects multiple `GPULICENSE` devices instead of choosing a token arbitrarily. + +Missing, empty, symlinked, nonregular or ambiguous media fails explicitly after stale managed-token cleanup. Unrelated local `gridd.conf` records, including UTF-8 comments, retain their bytes under `LC_ALL=C`; a missing final newline is added. Local text may contain tabs and LF/CRLF line endings but not NUL or other control bytes. Active exact managed-key assignments are replaced when importing a profile, without changing similarly named keys. The default `/etc/nvidia/ClientConfigToken` directory is not deleted or used. Existing configuration permissions are preserved; a new config is `0644`, runtime directories are root-owned `0755`, private staging is `0700` and the token is root-owned `0744`, following NVIDIA's documented mode. The token directory holds no config or lock file. The guest administrator and the licensing service can read the token; this is not root-only storage. + +The script unmounts only media it mounted. An occupied mountpoint, unsafe managed path, cleanup failure, non-memory backing, copy failure or unmount failure is an error. It never deliberately writes token bytes to the root disk or a disk-backed temporary file. Memory-backed does not mean that bytes can never reach persistent storage: [tmpfs can swap](https://docs.kernel.org/filesystems/tmpfs.html), and hibernation, core dumps and memory snapshots can capture them. A strict no-disk guarantee requires a separately verified image/operator policy for all those mechanisms; the script does not disable them. It also cannot prevent a guest administrator from copying a token. + +## Windows image installation + +Use the NVIDIA default token directory; custom registry token paths are outside this reference. The script does not import the disk's `gridd.conf` or write `FeatureType` into the registry. Confirm that your driver, licensing settings and certificate trust are valid before preparing the image. + +Install the verified script as an administrator, with a root-owned equivalent Windows ACL: SYSTEM and Administrators may modify it; ordinary users may not. Configure script signing and execution policy through your image's normal policy. Do not weaken execution policy to run this reference. For example, after placing the verified script at `C:\ProgramData\Cozystack\Set-CozystackVgpuLicense.ps1` and applying the required signature and ACL: + +```powershell +$Script = 'C:\ProgramData\Cozystack\Set-CozystackVgpuLicense.ps1' +$PowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' +$Action = New-ScheduledTaskAction -Execute $PowerShell -Argument "-NoProfile -NonInteractive -File `"$Script`"" +$Trigger = New-ScheduledTaskTrigger -AtStartup +$Principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +Register-ScheduledTask -TaskName 'Cozystack-vGPU-License' -Action $Action -Trigger $Trigger -Principal $Principal -Force +``` + +The task uses the Windows PowerShell executable supplied with Windows; PowerShell 7 is not a guest prerequisite. + +**An AtStartup task does not guarantee execution before `NvDisplayContainer`.** The service may already have read a persistent token before cleanup. Replacing the named task is idempotent and needs no user logon, but neither registration nor mock tests prove real boot ordering or revocation. This reference makes no Linux-equivalent startup guarantee on Windows. + +On every invocation, including without media, the script clears the contents of `%SystemDrive%\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken`. It creates a missing directory, rejects unsafe directory/reparse entries, optionally copies the single media token, and restarts `NvDisplayContainer`. Invalid media or a failed copy leaves no new token after successful cleanup, restarts the client and then reports failure. Cleanup failures and service failures are errors, not successful licensing. Monitor the task's result through your image operations; do not record token payloads in transcripts or error reports. + +## License server connectivity and driver compatibility + +The hoster chooses CLS or DLS and DLS placement, including a management cluster. From the guest, verify DNS resolution, routing, TLS trust and access to the endpoints named by its token. Check the actual DLS deployment's listening ports and the client's requirements in NLS §2.4; do not use a remembered port list as an allow-list. An in-cluster DLS may need explicit tenant network-policy exceptions under the installed CNI. Current isolation defaults are not an architectural prohibition on that placement. + +If acquisition or renewal fails, check the guest client logs, available licenses, endpoint reachability and guest configuration. Inspect `nvidia-smi -q` for license status with the product's enforcement mode in mind: it is not independently a server-side lease audit. A connectivity fix may need a client restart; a platform token change needs a new VMI. NVIDIA's release-specific grace and degradation rules apply. A recovered guest can regain full capability, although applications started while degraded may require restarting. + +Before upgrading host vGPU Manager, check every guest driver against the applicable release's compatibility table. NVIDIA AI Enterprise and NVIDIA vGPU software have different release lines; do not compare their version numbers as if they were one product. An entitled tenant can update a compatible driver **inside the guest**. Recreating the VM from an updated image is an alternative, not a requirement. Confirm NVIDIA support for the actual host/guest/compute combination and distribution rights before offering driver-bearing images. + +## Usage, rotation and revocation + +Platform usage records describe GPU use per tenant while launcher pods exist. They are not active license counts: Cozystack does not query the license server. A stopped VM can retain a lease until expiry, and an Offline Lease can survive a clean shutdown. Reconcile tenant usage and server leases in the hoster's billing process. + +The accepted rotation procedure moves to a new allocation or license server instance, replaces the source Secret token and restarts each VMI. An OS reboot inside the same VMI does not refresh its Secret disk. Do not assume that editing a Secret updates a running guest. Plan spare capacity on the new allocation, a restart deadline, and when to disable leasing on the old allocation. Check the configured maximum lease duration: disabling leasing does not immediately erase existing leases. + +A token's expiry is distinct from a lease's expiry. NLS 4.0.0 documents a default token expiry of 12 years and a DLS no-expiry option (§4.4); choose an explicit lifetime compatible with deployed drivers. Generating a replacement token alone is not revocation. The guide documents no method to revoke an individual token before expiry. Before recommending same-instance DLS allocation rotation operationally, verify that disabling the old allocation prevents its token from obtaining a new license; that live check is not supplied by these scripts. + +Manual release frees a lease, not the ability to request another one with a still-served token. NLS §4.9 documents release limits; check the limit shown by your actual CLS/DLS before use rather than relying on a fixed number here. Clearing platform entitlement blocks subsequent class-based starts and removes namespace copies; running guests keep running and may renew while their allocation serves them. Immediate cut-off also requires stopping the affected vGPU VMs and addressing copied-token reuse through the server-side rotation procedure. One class token is shared across entitled namespaces, not a per-tenant revocation mechanism. + +## Security + +Do not put platform tokens in golden images, cloud-init, bootstrap data, chart values, examples, CI fixtures or diagnostic bundles. Build from a token-free source image: installing this reference does not scrub old tokens from a previous image, cloud-init data or snapshots. Treat previously distributed tokens as exposed and include them in the rotation plan. An entitled guest administrator can read and retain its token by design. Memory-backed Linux placement limits ordinary root-disk persistence, not deliberate copying or memory capture. + +On Windows the token persists on the system disk and can enter snapshots, clones and exports. The licensing service may use a previous token before the startup task runs. Cleanup and restart prove neither immediate lease release nor token revocation, and they do not provide Linux-equivalent stale-token protection. BYOL credentials remain tenant-owned and must not be exposed to these cleanup references. + +## Guest acceptance checklist + +Offline tests execute the canonical Linux script in a synthetic filesystem with stubbed OS commands, check Linux unit dependency structure, and exercise the Windows script under Pester with mocked volume and service calls. They do not access a GPU, NVIDIA binary, cluster, real token or DLS/CLS. Windows mocks do not prove loaded-token removal, licensing state or real boot order. + +Run each checklist separately on disposable guests with permission to use the driver, GPU and license. Record the reference commit, guest OS, systemd/Windows, driver and license-server versions, sanitized operation order and observed license state. Never save token bytes, Secret dumps, private endpoints or raw transcripts. Keep timestamps with their source timezone and the operator's local timezone. + +### Linux + +- [ ] First boot with media: confirm a valid NVIDIA lease, memory backing, token-only directory and modes, merged vendor config, and journal ordering before the real licensing client. +- [ ] New VMI without media: confirm empty managed storage, no default-path fallback and an otherwise bootable VM. +- [ ] Malformed media: confirm an explicit preparation error and a blocked client, without blocking VM boot. +- [ ] Rotate synthetic-labelled token A to token B through a new VMI and observe actual lease behaviour without exporting token contents. +- [ ] Test unreachable endpoint and recovery, then a denied class-based start without entitlement. +- [ ] Check guest root disk, swap and snapshot policy for unintended token persistence. + +### Windows + +- [ ] First boot with media: observe the scheduled task, folder contents and real licensing result. +- [ ] New VMI without media and a stale-token fixture: observe cleanup and the real task/service timeline, including a service that already started. Do not infer pre-service ordering. +- [ ] Malformed media and copy/cleanup/restart failures: observe reported errors and resulting folder/service state. +- [ ] Rotate A to B through a new VMI; observe actual lease lifecycle, unreachable endpoint and recovery, and denied starts without entitlement. +- [ ] Review snapshots, clones and exports for persistent token exposure. + +Claims about immediate revocation require additional real-server observations: release/renewal, old-token reuse, configured expiry and maximum-lease boundaries. A successful script run or client restart is insufficient evidence. + +## Sources and version context + +Vendor client operations were checked on 9 November 2026 against the [NVIDIA License System User Guide](https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html), whose page metadata identifies **NLS 4.0.0**, last updated 19 October 2026. Sections §5.1.1, §5.1.2 and §5.1.3.2 document the Windows directory/service, Linux token mode, custom token directory and token-only requirement. The `latest` URL moves; recheck these facts when changing deployed versions. + +For product-specific enforcement and compatibility, use the deployed release's [NVIDIA vGPU licensing guide](https://docs.nvidia.com/vgpu/latest/grid-licensing-user-guide/), [Linux/KVM release notes](https://docs.nvidia.com/vgpu/latest/grid-vgpu-release-notes-generic-linux-kvm/), and [NVIDIA AI Enterprise documentation](https://docs.nvidia.com/ai-enterprise/). This reference does not substitute an old product's grace-period numbers or compatibility table for the release you run. diff --git a/content/en/docs/next/virtualization/vgpu.md b/content/en/docs/next/virtualization/vgpu.md index ed2a017fa..fc62167f7 100644 --- a/content/en/docs/next/virtualization/vgpu.md +++ b/content/en/docs/next/virtualization/vgpu.md @@ -1,7 +1,7 @@ --- title: "NVIDIA vGPU for Virtual Machines" linkTitle: "vGPU" -description: "Slice one physical NVIDIA GPU across several virtual machines with the vgpu variant of the GPU Operator package, including SR-IOV profile assignment and DLS licensing." +description: "Slice one physical NVIDIA GPU across several virtual machines with the vgpu variant of the GPU Operator package, including SR-IOV profile assignment and NVIDIA License System configuration." weight: 45 --- @@ -24,7 +24,7 @@ This guide focuses on the **SR-IOV path**, which is the only model NVIDIA suppor - Ubuntu 24.04 host OS. Older Ubuntu releases also work if the upstream `gpu-driver-container` repository has a matching `vgpu-manager/` Dockerfile. **Talos Linux is not recommended** for vGPU: NVIDIA does not publicly distribute the vGPU guest driver — it requires NVIDIA Enterprise Portal access — and Sidero [closed siderolabs/extensions#461](https://github.com/siderolabs/extensions/issues/461) noting that they cannot support vGPU "unless NVIDIA changes their licensing terms or provides us a way to obtain, test, and distribute the software". Building a Talos system extension that includes the driver in-tree is therefore not feasible without a private fork that violates the EULA. - KubeVirt v1.9.0 or newer, which is the release the platform bundles. [kubevirt/kubevirt#16890](https://github.com/kubevirt/kubevirt/pull/16890) ("vGPU: SRIOV support") shipped in v1.9.0; v1.8.x and older do not include it and backports are not planned. - An NVIDIA vGPU Software or NVIDIA AI Enterprise subscription (the `.run` is not redistributable). -- A reachable NVIDIA Delegated License Service (DLS) instance and a matching `client_configuration_token.tok` file. +- Reachable NVIDIA Cloud License Service (CLS) or Delegated License Service (DLS) endpoints and a matching client configuration token. ## Variants @@ -169,47 +169,15 @@ On L40S and other Ada Lovelace cards the SR-IOV VFs report the same PCI device I kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{": "}{.status.allocatable.nvidia\.com/L40S-24Q}{"\n"}{end}' ``` -## Licensing (DLS) +## Licensing (CLS or DLS) {#licensing-dls} -vGPU 17/20 uses the NVIDIA Delegated License Service. The legacy `ServerAddress=` and `ServerPort=7070` lines in `gridd.conf` are no longer authoritative — `nvidia-gridd`, running **inside the guest**, reads the DLS endpoint from the ClientConfigToken file directly. +NVIDIA License System supports CLS or DLS. The licensing client running inside the guest reads its server endpoints from the client configuration token. The host vGPU Manager DaemonSet does not request a license — it only enables SR-IOV and loads `nvidia.ko`. Licensing is consumed entirely by the guest. The gpu-operator chart's `driver.licensingConfig.secretName` would mount the Secret into the **driver pod on the host**, where it has no effect for SR-IOV vGPU; do not wire the licensing Secret through it. -Instead, deliver the token and `gridd.conf` to the guest via cloud-init or a containerDisk overlay: +For platform-licensed GPU classes, use the [guest licensing reference]({{% ref "/docs/next/virtualization/vgpu-licensing" %}}). The platform delivers `client_configuration_token.tok` and `gridd.conf` on a disk labelled `GPULICENSE`. The Linux reference prepares a token-only memory-backed directory before the licensing client; the Windows reference has persistent-token and startup-order limitations described in that guide. Rotation requires a new VMI, not an OS reboot inside the same VMI. -```yaml -# inside the VirtualMachine cloudInitNoCloud userData -write_files: -- path: /etc/nvidia/ClientConfigToken/client_configuration_token.tok - # 0744 follows NVIDIA's recommendation in the Virtual GPU Software - # Licensing User Guide ("Configuring a Licensed Client on Linux"): - # nvidia-gridd does not necessarily run as the file owner. - # https://docs.nvidia.com/vgpu/latest/grid-licensing-user-guide/ - permissions: '0744' - encoding: b64 - content: -- path: /etc/nvidia/gridd.conf - permissions: '0644' - content: | - # FeatureType selects which vGPU Software license the guest requests. - # 0 — unlicensed state (no license requested; Q profiles run in - # reduced mode after the grace period). - # 1 — NVIDIA vGPU. The driver auto-selects the correct license - # type from the configured vGPU profile (Q → vWS, B → vPC, - # A → vCS / Compute). Use this for SR-IOV vGPU profiles. - # 2 — explicitly NVIDIA RTX Virtual Workstation. - # 4 — explicitly NVIDIA Virtual Compute Server. - FeatureType=1 -``` - -Verify activation inside the guest: - -```bash -nvidia-smi -q | grep 'License Status' -# License Status : Licensed -``` - -If the guest reports `Unlicensed (Unrestricted)` for more than a couple of minutes, check `journalctl _COMM=nvidia-gridd` for handshake errors against the DLS endpoint baked into the token. +Do not put platform tokens in cloud-init, containerDisk overlays or shared images. A BYOL class has no platform `license`; its tenant configures licensing separately, in an image that does not run the platform reference unit or startup task. ### Migrating from chart v25.x diff --git a/hack/Test-VgpuGuest.Tests.ps1 b/hack/Test-VgpuGuest.Tests.ps1 new file mode 100644 index 000000000..24534ee7a --- /dev/null +++ b/hack/Test-VgpuGuest.Tests.ps1 @@ -0,0 +1,212 @@ +BeforeAll { + $Asset = Join-Path $PSScriptRoot '../static/examples/vgpu-guest/v1/windows/Set-CozystackVgpuLicense.ps1' + # Keep cmdlet objects: module-qualified names can still resolve to Pester mocks. + $RemoveItemCmdlet = Get-Command -Name Remove-Item -Module Microsoft.PowerShell.Management -CommandType Cmdlet -All -ErrorAction Stop + $CopyItemCmdlet = Get-Command -Name Copy-Item -Module Microsoft.PowerShell.Management -CommandType Cmdlet -All -ErrorAction Stop + + function Remove-GuestFixture([string]$Path, [switch]$Recurse) { + if (-not $Path.StartsWith($Root + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { + throw 'Fixture cleanup outside owned TestDrive root' + } + & $RemoveItemCmdlet -LiteralPath $Path -Recurse:$Recurse -Force -ErrorAction Stop + } +} + +Describe 'Platform vGPU guest token' { + BeforeEach { + $Root = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + # Same separators as the script: the mocks compare paths as strings. + $Folder = Join-Path $Root 'Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken' + $Media = Join-Path $Root 'media with spaces' + New-Item -ItemType Directory -Path $Folder, $Media -Force | Out-Null + Set-Content -LiteralPath (Join-Path $Folder 'old.tok') -Value 'SYNTHETIC-STALE' + Set-Content -LiteralPath (Join-Path $Media 'client_configuration_token.tok') -Value 'SYNTHETIC-TOKEN-A' + Set-Content -LiteralPath (Join-Path $Media 'gridd.conf') -Value 'FeatureType=4' + $env:COZYSTACK_VGPU_TEST_ROOT = $Root + $Events = [System.Collections.Generic.List[string]]::new() + Mock Get-Volume { [pscustomobject]@{ FileSystemLabel = 'GPULICENSE'; Path = $Media } } + Mock Remove-Item { + if (-not $LiteralPath.StartsWith($Folder + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { + throw 'Deletion outside TestDrive token directory' + } + $Events.Add('cleanup') + & $RemoveItemCmdlet -LiteralPath $LiteralPath -Force -ErrorAction Stop + } + Mock Copy-Item { + if (-not $Destination.StartsWith($Folder + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { + throw 'Copy outside TestDrive token directory' + } + $Events.Add('copy') + & $CopyItemCmdlet -LiteralPath $LiteralPath -Destination $Destination -ErrorAction Stop + } + $Client = @{ State = 'Running'; Restarts = 0 } + Mock Restart-Service { + $Client.State | Should -Be 'Running' + $Events.Add('restart') + $Client.State = 'Restarted' + $Client.Restarts++ + } + Mock Set-ItemProperty { throw 'Registry writes are forbidden' } + } + + AfterEach { + $env:COZYSTACK_VGPU_TEST_ROOT = $null + } + + It 'cleans an already-running client folder, copies only token, then restarts the actual client' { + & $Asset -TestRoot $Root + @($Events) -join ',' | Should -Be 'cleanup,copy,restart' + (@(Get-ChildItem -LiteralPath $Folder).Name -join ',') | Should -Be 'client_configuration_token.tok' + Get-Content -LiteralPath (Join-Path $Folder 'client_configuration_token.tok') | Should -Be 'SYNTHETIC-TOKEN-A' + Should -Invoke Restart-Service -Times 1 -Exactly -ParameterFilter { $Name -eq 'NvDisplayContainer' } + Should -Invoke Set-ItemProperty -Times 0 + $Client.State | Should -Be 'Restarted' + $Client.Restarts | Should -Be 1 + $Client.State = 'Running' + & $Asset -TestRoot $Root + $Client.State | Should -Be 'Restarted' + $Client.Restarts | Should -Be 2 + Should -Invoke Restart-Service -Times 2 -Exactly + } + + It 'cleans stale tokens and restarts when media is absent' { + Mock Get-Volume { @() } + & $Asset -TestRoot $Root + @(Get-ChildItem -LiteralPath $Folder).Count | Should -Be 0 + @($Events) -join ',' | Should -Be 'cleanup,restart' + } + + It 'creates a missing token directory and restarts without media' { + Remove-GuestFixture -Path $Folder -Recurse + Mock Get-Volume { @() } + & $Asset -TestRoot $Root + Test-Path -LiteralPath $Folder -PathType Container | Should -BeTrue + @($Events) -join ',' | Should -Be 'restart' + } + + It 'rejects ambiguous media after cleanup and still restarts' { + Mock Get-Volume { @([pscustomobject]@{ FileSystemLabel = 'GPULICENSE'; Path = $Media }) * 2 } + { & $Asset -TestRoot $Root } | Should -Throw '*vGPU license media preparation failed*' + @(Get-ChildItem -LiteralPath $Folder).Count | Should -Be 0 + @($Events) -join ',' | Should -Be 'cleanup,restart' + } + + It 'fails closed for a token' -ForEach @( + @{ Kind = 'missing' }, @{ Kind = 'empty' }, @{ Kind = 'directory' }, @{ Kind = 'symlink' } + ) { + $token = Join-Path $Media 'client_configuration_token.tok' + Remove-GuestFixture -Path $token + switch ($Kind) { + empty { New-Item -ItemType File -Path $token | Out-Null } + directory { New-Item -ItemType Directory -Path $token | Out-Null } + symlink { New-Item -ItemType Junction -Path $token -Target $Folder | Out-Null } + } + { & $Asset -TestRoot $Root } | Should -Throw '*vGPU license media preparation failed*' + @(Get-ChildItem -LiteralPath $Folder).Count | Should -Be 0 + Should -Invoke Copy-Item -Times 0 + Should -Invoke Restart-Service -Times 1 -Exactly + } + + It 'removes partial copies before restart and emits no payload on copy failure' { + Mock Copy-Item { + Set-Content -LiteralPath $Destination -Value 'SYNTHETIC-PARTIAL' + throw 'SYNTHETIC-SECRET-ERROR' + } + try { + & $Asset -TestRoot $Root + throw 'Expected failure was missing' + } catch { + $_.Exception.Message | Should -Be 'vGPU license media preparation failed' + } + @(Get-ChildItem -LiteralPath $Folder).Count | Should -Be 0 + Should -Invoke Restart-Service -Times 1 -Exactly + } + + It 'reports cleanup failure without copying or claiming a restart succeeded' { + Mock Remove-Item { throw 'SYNTHETIC-SECRET-ERROR' } + { & $Asset -TestRoot $Root } | Should -Throw '*vGPU license token cleanup failed*' + Should -Invoke Remove-Item -Times 1 -Exactly + Should -Invoke Copy-Item -Times 0 + Should -Invoke Restart-Service -Times 0 + } + + It 'reports a real locked-file cleanup error before copy or restart' { + $stale = Join-Path $Folder 'old.tok' + $handle = [IO.File]::Open($stale, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::None) + try { + { & $Asset -TestRoot $Root } | Should -Throw '*vGPU license token cleanup failed*' + Test-Path -LiteralPath $stale | Should -BeTrue + Should -Invoke Remove-Item -Times 1 -Exactly + Should -Invoke Copy-Item -Times 0 + Should -Invoke Restart-Service -Times 0 + } finally { + $handle.Dispose() + } + } + + It 'reports service restart failure' { + Mock Restart-Service { throw 'SYNTHETIC-SECRET-ERROR' } + { & $Asset -TestRoot $Root } | Should -Throw '*vGPU license client restart failed*' + } + + It 'rejects fixture cleanup outside the owned TestDrive subdirectory' { + $outside = Join-Path $TestDrive ([guid]::NewGuid().ToString() + '.tok') + Set-Content -LiteralPath $outside -Value 'SYNTHETIC-OUTSIDE' + { Remove-GuestFixture -Path $outside } | Should -Throw '*Fixture cleanup outside owned TestDrive root*' + Get-Content -LiteralPath $outside | Should -Be 'SYNTHETIC-OUTSIDE' + } + + It 'rejects a foreign sandbox before filesystem or service operations' { + { & $Asset -TestRoot 'C:\' } | Should -Throw '*Invalid sandbox invocation*' + Should -Invoke Remove-Item -Times 0 + Should -Invoke Restart-Service -Times 0 + } +} + +Describe 'Nonempty Windows CI selection' { + BeforeAll { + $workflow = Get-Content -Raw -LiteralPath (Join-Path $PSScriptRoot '../.github/workflows/vgpu-guest.yml') + $guard = [regex]::Match($workflow, '(?ms)^ if \(\$null -eq \$result.*?^ \}') + if (-not $guard.Success) { throw 'Missing nonempty Pester selection guard' } + $SelectionGuard = [scriptblock]::Create($guard.Value) + } + + It 'rejects a run with tests and passes' -ForEach @( + @{ Total = 0; Passed = 0 }, @{ Total = 4; Passed = 0 } + ) { + $result = [pscustomobject]@{ TotalCount = $Total; PassedCount = $Passed } + { & $SelectionGuard } | Should -Throw '*No vGPU guest tests passed*' + } + + It 'accepts a nonempty passing run' { + $result = [pscustomobject]@{ TotalCount = 1; PassedCount = 1 } + { & $SelectionGuard } | Should -Not -Throw + } +} + +Describe 'Documented Windows startup task wiring' { + It 'registers only the owned task under SYSTEM at startup without a policy bypass' { + Mock New-ScheduledTaskAction { [Microsoft.Management.Infrastructure.CimInstance]::new('MSFT_TaskAction', 'root/Microsoft/Windows/TaskScheduler') } + Mock New-ScheduledTaskTrigger { [Microsoft.Management.Infrastructure.CimInstance]::new('MSFT_TaskTrigger', 'root/Microsoft/Windows/TaskScheduler') } + Mock New-ScheduledTaskPrincipal { [Microsoft.Management.Infrastructure.CimInstance]::new('MSFT_TaskPrincipal', 'root/Microsoft/Windows/TaskScheduler') } + Mock Register-ScheduledTask { } + $guide = Get-Content -Raw -LiteralPath (Join-Path $PSScriptRoot '../content/en/docs/next/virtualization/vgpu-licensing.md') + $blocks = [regex]::Matches($guide, '(?s)```powershell\r?\n(.*?)```') + $task = @($blocks | Where-Object { $_.Groups[1].Value.Contains('Register-ScheduledTask') }) + $task.Count | Should -Be 1 + $code = $task[0].Groups[1].Value + $code | Should -Not -Match 'ExecutionPolicy|Bypass|Unregister-ScheduledTask' + & ([scriptblock]::Create($code)) | Out-Null + Should -Invoke New-ScheduledTaskAction -Times 1 -Exactly -ParameterFilter { + $Execute -eq (Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe') -and + $Argument -eq '-NoProfile -NonInteractive -File "C:\ProgramData\Cozystack\Set-CozystackVgpuLicense.ps1"' + } + Should -Invoke New-ScheduledTaskTrigger -Times 1 -Exactly -ParameterFilter { $AtStartup } + Should -Invoke New-ScheduledTaskPrincipal -Times 1 -Exactly -ParameterFilter { + $UserId -eq 'SYSTEM' -and $LogonType -eq 'ServiceAccount' -and $RunLevel -eq 'Highest' + } + Should -Invoke Register-ScheduledTask -Times 1 -Exactly -ParameterFilter { + $TaskName -eq 'Cozystack-vGPU-License' -and $Force + } + } +} diff --git a/hack/test_vgpu_guest.py b/hack/test_vgpu_guest.py new file mode 100644 index 000000000..e4c0bc14c --- /dev/null +++ b/hack/test_vgpu_guest.py @@ -0,0 +1,468 @@ +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +ASSETS = ROOT / "static/examples/vgpu-guest/v1/linux" +SCRIPT = ASSETS / "cozystack-vgpu-license.sh" +TOKEN = "client_configuration_token.tok" + + +class GuestLicense(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="vgpu-guest-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.runtime = self.root / "run/cozystack-vgpu-license" + self.tokens = self.runtime / "token" + self.media = self.runtime / "media" + self.conf = self.root / "etc/nvidia/gridd.conf" + self.conf.parent.mkdir(parents=True) + self.media.mkdir(parents=True) + self.tokens.mkdir() + self.outside = self.root / "outside" + self.outside.write_text("DO NOT CHANGE") + self.default = self.conf.parent / "ClientConfigToken" + self.default.mkdir() + (self.default / TOKEN).write_text("SYNTHETIC-BYOL-SENTINEL") + self.bin = self.root / "bin" + self.bin.mkdir() + self.log = self.root / "commands" + stub = f'#!{sys.executable} -I\n' + '''import os, pathlib, sys +root = pathlib.Path(os.environ["COZYSTACK_VGPU_TEST_ROOT"]).resolve() +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +with (root / "commands").open("a") as out: + out.write(name + " " + " ".join(args) + "\\n") +for arg in args: + if arg == "/dev/null" and name == "awk": continue + if arg.startswith("/") and not pathlib.Path(arg).is_relative_to(root): + sys.exit(90) +if os.environ.get("FAIL_COMMAND") == name: + sys.exit(1) +mode = os.environ.get("MEDIA", "one") +if name == "lsblk": + if "--nodeps" in args: + print("8:2" if os.environ.get("STALE_ALIAS") and args[-1].endswith("GPULICENSE") else "8:1") + else: + print(str(root / "root-device") + " ROOT") + print(str(root / "blank-data-device") + " ") + if mode != "none": print(str(root / "device") + " GPULICENSE") + if mode == "ambiguous": print(str(root / "other-device") + " GPULICENSE") +elif name == "blkid": + if "--probe" in args: + if os.environ.get("PROBE_STATUS"): sys.exit(int(os.environ["PROBE_STATUS"])) + print("DEVNAME=" + args[-1]) + print("TYPE=ext4" if args[-1].endswith("root-device") else os.environ.get("PROBE_LABEL", "LABEL=GPULICENSE")) + else: + if mode == "none": sys.exit(2) + print(root / "device") + if mode == "ambiguous": print(root / "other-device") +elif name == "findmnt": + if "--mountpoint" in args: + if not (root / "mounted").exists() or os.environ.get("NO_EXACT_MOUNT"): sys.exit(1) + column = args[args.index("--output") + 1] + if column == "TARGET": + print(root / "run") + if os.environ.get("PREMOUNTED"): print(root / "run/cozystack-vgpu-license/media") + elif column == "MAJ:MIN": + print(os.environ.get("MOUNT_IDENTITY", "8:1")) + elif column == "OPTIONS": + print(os.environ.get("MOUNT_OPTIONS", "ro,nodev,nosuid,noexec")) + elif column == "ID": + print("2" if os.environ.get("SPLIT_STAGE") and args[-1].endswith("/.stage") else "1") + elif args[-1].endswith("/token"): + print(os.environ.get("TOKEN_FSTYPE", os.environ.get("FSTYPE", "tmpfs"))) + else: + print(os.environ.get("FSTYPE", "tmpfs")) +elif name in ("cp", "mv", "chmod", "rm", "od", "awk"): + if os.environ.get("PARTIAL_COPY") and name == "cp" and args[-2].endswith(".tok"): + pathlib.Path(args[-1]).write_bytes(b"SYNTHETIC-PARTIAL") + sys.exit(1) + os.execv(os.environ["REAL_" + name.upper()], [name] + args) +elif name == "mount": + (root / "mounted").touch() +elif name == "umount": + (root / "mounted").unlink() +elif name != "udevadm": + sys.exit(91) +''' + commands = ("blkid", "lsblk", "udevadm", "mount", "umount", "findmnt", "systemctl", "service", + "cp", "mv", "chmod", "rm", "od", "awk") + self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}", + COZYSTACK_VGPU_TEST_ROOT=str(self.root)) + for name in commands: + executable = shutil.which(os.environ.get("VGPU_TEST_AWK", "awk") if name == "awk" else name) + if executable: + self.env["REAL_" + name.upper()] = executable + path = self.bin / name + path.write_text(stub) + path.chmod(0o755) + for name in ("device", "other-device", "root-device", "blank-data-device"): + (self.root / name).touch() + self.valid_media() + + def valid_media(self, feature="1", token=b"SYNTHETIC-OPAQUE-TOKEN-A\n"): + (self.media / TOKEN).write_bytes(token) + (self.media / "gridd.conf").write_text(f"FeatureType={feature}\nClientConfigTokenPath=/ignored\n") + + def run_script(self, ok=True, **env): + result = subprocess.run(["sh", str(SCRIPT), "--test-root", str(self.root)], + env=dict(self.env, **env), capture_output=True, text=True) + self.assertEqual(result.returncode == 0, ok, result.stderr) + self.assertNotIn("SYNTHETIC-", result.stdout + result.stderr) + self.assertEqual(self.outside.read_text(), "DO NOT CHANGE") + self.assertEqual((self.default / TOKEN).read_text(), "SYNTHETIC-BYOL-SENTINEL") + if self.log.exists(): + self.assertNotIn("systemctl ", self.log.read_text()) + self.assertNotIn("service ", self.log.read_text()) + return result + + def assert_empty(self): + self.assertEqual(list(self.tokens.iterdir()), []) + + def test_license_with_blank_data_disk_probes_only_candidate(self): + self.run_script() + commands = self.log.read_text() + self.assertIn("udevadm settle --timeout=10", commands) + self.assertIn("lsblk --list --noheadings --paths --raw --output NAME,LABEL", commands) + probes = [line for line in commands.splitlines() if line.startswith("blkid ")] + self.assertEqual(probes, [f"blkid --probe --output export --match-tag LABEL -- {self.root / 'device'}"]) + + def test_probe_status_two_is_failure_not_absence(self): + (self.tokens / TOKEN).write_bytes(b"SYNTHETIC-STALE") + self.run_script(ok=False, PROBE_STATUS="2") + self.assert_empty() + + def test_stale_udev_label_link_is_failure(self): + alias = self.root / "dev/disk/by-label/GPULICENSE" + alias.parent.mkdir(parents=True) + alias.symlink_to(self.root / "missing-device") + self.run_script(ok=False, MEDIA="none") + self.assert_empty() + + def test_label_alias_agrees_with_inventory_and_fresh_probe(self): + alias = self.root / "dev/disk/by-label/GPULICENSE" + alias.parent.mkdir(parents=True) + alias.symlink_to(self.root / "device") + self.run_script() + for env in ({"STALE_ALIAS": "yes"}, {"PROBE_LABEL": "LABEL=OTHER"}, {"PROBE_STATUS": "8"}): + with self.subTest(env=env): + self.run_script(ok=False, **env) + self.assert_empty() + + def test_complete_discovery_required(self): + for command in ("udevadm", "lsblk"): + with self.subTest(command=command): + (self.tokens / TOKEN).write_bytes(b"SYNTHETIC-STALE") + self.run_script(ok=False, FAIL_COMMAND=command) + self.assert_empty() + + def test_first_boot_modes_and_rotation(self): + self.run_script() + self.assertEqual((self.tokens / TOKEN).read_bytes(), (self.media / TOKEN).read_bytes()) + self.assertEqual(stat.S_IMODE((self.tokens / TOKEN).stat().st_mode), 0o744) + self.assertEqual(stat.S_IMODE(self.tokens.stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE(self.conf.stat().st_mode), 0o644) + self.assertEqual(list(self.tokens.iterdir()), [self.tokens / TOKEN]) + self.assertIn("--options ro,nodev,nosuid,noexec", self.log.read_text()) + self.assertEqual(self.log.read_text().count("umount "), 1) + self.valid_media("4", b"SYNTHETIC-OPAQUE-TOKEN-B") + self.run_script() + self.assertEqual((self.tokens / TOKEN).read_bytes(), b"SYNTHETIC-OPAQUE-TOKEN-B") + self.assertIn("FeatureType=4", self.conf.read_text()) + self.run_script() + self.assertEqual(self.conf.read_text().count("FeatureType="), 1) + + def test_merge_preserves_vendor_settings(self): + self.conf.write_text("# FeatureType=9\nEnableUI=TRUE\nFeatureType=9\nFeatureType = 2\nClientConfigTokenPath=/old\nClientConfigTokenPath=/older\nOtherFeatureType=7\n") + self.run_script() + self.assertEqual(self.conf.read_text(), f"# FeatureType=9\nEnableUI=TRUE\nOtherFeatureType=7\nClientConfigTokenPath={self.tokens}\nFeatureType=1\n") + + def test_no_media_succeeds_with_empty_managed_path(self): + self.conf.write_text("FeatureType=4\nClientConfigTokenPath=/stale\n") + (self.tokens / TOKEN).write_text("SYNTHETIC-STALE") + self.run_script(MEDIA="none") + self.assert_empty() + self.assertIn(f"ClientConfigTokenPath={self.tokens}", self.conf.read_text()) + self.assertIn("FeatureType=4", self.conf.read_text()) + self.assertNotIn("mount ", self.log.read_text()) + self.assertNotIn("blkid ", self.log.read_text()) + + def test_missing_runtime_and_config(self): + shutil.rmtree(self.runtime) + self.run_script(MEDIA="none") + self.assert_empty() + self.assertEqual(self.conf.read_text(), f"ClientConfigTokenPath={self.tokens}\n") + + def test_invalid_media_fails_after_stale_cleanup(self): + variants = ("missing", "empty", "directory", "symlink", "config-missing", "config-symlink", + "fifo", "config-fifo", "feature-missing", "feature-invalid", "feature-newline", "feature-duplicate", "feature-command") + for variant in variants: + with self.subTest(variant=variant): + for item in self.media.iterdir(): + if item.is_dir(): item.rmdir() + else: item.unlink() + self.valid_media() + token = self.media / TOKEN + conf = self.media / "gridd.conf" + if variant in ("missing", "empty", "directory", "symlink", "fifo"): + token.unlink() + if variant == "empty": token.touch() + if variant == "directory": token.mkdir() + if variant == "symlink": token.symlink_to(self.outside) + if variant == "fifo": os.mkfifo(token) + elif variant == "config-fifo": + conf.unlink() + os.mkfifo(conf) + elif variant == "config-missing": conf.unlink() + elif variant == "config-symlink": + conf.unlink() + conf.symlink_to(self.outside) + else: + conf.write_text({"feature-missing": "Other=1\n", "feature-invalid": "FeatureType=2\n", + "feature-newline": "FeatureType=\n1\n", + "feature-duplicate": "FeatureType=1\nFeatureType=4\n", + "feature-command": "FeatureType=$(touch /outside)\n"}[variant]) + (self.tokens / TOKEN).write_text("SYNTHETIC-STALE") + self.run_script(ok=False) + self.assert_empty() + self.assertIn(f"ClientConfigTokenPath={self.tokens}", self.conf.read_text()) + + def test_ambiguity_mount_failure_and_foreign_mount(self): + for env in ({"MEDIA": "ambiguous"}, {"FAIL_COMMAND": "mount"}, {"PREMOUNTED": "yes"}): + with self.subTest(env=env): + self.log.write_text("") + self.run_script(ok=False, **env) + self.assert_empty() + self.assertNotIn("umount ", self.log.read_text()) + + def test_unmount_failure_is_failure_and_removes_token(self): + self.run_script(ok=False, FAIL_COMMAND="umount") + self.assert_empty() + + def test_memory_filesystem_required_before_token_write(self): + self.run_script(ok=False, FSTYPE="ext4") + self.assert_empty() + self.assertFalse(self.conf.exists()) + self.assertNotIn("mount ", self.log.read_text()) + + def test_token_child_mount_must_also_be_memory_backed(self): + self.run_script(ok=False, TOKEN_FSTYPE="ext4") + self.assert_empty() + self.assertNotIn("mount ", self.log.read_text()) + + def test_runtime_symlink_and_unowned_subdirectory_rejected(self): + self.tokens.rmdir() + self.tokens.symlink_to(self.default) + self.run_script(ok=False) + self.tokens.unlink() + self.tokens.mkdir() + (self.tokens / "foreign-directory").mkdir() + self.run_script(ok=False) + self.assertTrue((self.tokens / "foreign-directory").is_dir()) + + @unittest.skipIf(os.geteuid() == 0, "Permission failures require an unprivileged test process") + def test_token_read_and_copy_failures_leave_no_token(self): + (self.media / TOKEN).chmod(0) + try: + self.run_script(ok=False) + self.assert_empty() + finally: + (self.media / TOKEN).chmod(0o644) + self.tokens.chmod(0o555) + try: + self.run_script(ok=False) + self.assert_empty() + finally: + self.tokens.chmod(0o755) + + def test_discovery_and_filesystem_probe_errors_are_not_no_media(self): + for command in ("blkid", "findmnt"): + with self.subTest(command=command): + self.run_script(ok=False, FAIL_COMMAND=command) + self.assert_empty() + + def test_posix_shell_without_guest_python(self): + self.assertEqual(SCRIPT.read_text().splitlines()[0], "#!/bin/sh") + self.assertNotIn("python", SCRIPT.read_text().lower()) + + def test_local_utf8_comments_crlf_and_missing_final_newline(self): + original = "# Mémoire — 設定\r\nVendor=✓\r\nFeatureTypeExtra=7" + self.conf.write_bytes(original.encode()) + self.run_script() + self.assertTrue(self.conf.read_bytes().startswith(original.encode() + b"\n")) + + def test_local_config_rejects_binary_and_bare_carriage_returns(self): + for content in (b"Vendor=\x00\n", b"Vendor=yes\rBAD\n", b"Vendor=yes\r"): + with self.subTest(content=content): + self.conf.write_bytes(content) + self.run_script(ok=False) + self.assert_empty() + self.assertEqual(self.conf.read_bytes(), content) + + def test_opaque_binary_token_and_media_crlf(self): + data = bytes(range(256)) + b"\x00\n\xff" + self.valid_media(token=data) + (self.media / "gridd.conf").write_bytes(b"# generated\r\n\tFeatureType \t= 4\t\r\n") + self.run_script() + self.assertEqual((self.tokens / TOKEN).read_bytes(), data) + self.assertIn("FeatureType=4\n", self.conf.read_text()) + + def test_malformed_text_and_identical_feature_duplicates_fail(self): + for content in (b"FeatureType=1\nFeatureType=1\n", b"FeatureType=1\x00\n", + b"FeatureType=1 4\n", b"FeatureType=1 # comment\n", b"FeatureType=1\rjunk\n", + b"#\xff\nFeatureType=1\n"): + with self.subTest(content=content): + (self.media / "gridd.conf").write_bytes(content) + self.run_script(ok=False) + self.assert_empty() + + def test_staging_must_share_token_mount_and_effective_readonly_options(self): + for env in ({"SPLIT_STAGE": "yes"}, {"MOUNT_OPTIONS": "rw,nodev,nosuid,noexec"}): + with self.subTest(env=env): + self.run_script(ok=False, **env) + self.assert_empty() + + def test_exact_mountpoint_and_candidate_identity_required(self): + for env in ({"NO_EXACT_MOUNT": "yes"}, {"MOUNT_IDENTITY": "8:2"}): + with self.subTest(env=env): + self.log.write_text("") + self.run_script(ok=False, **env) + self.assert_empty() + self.assertIn("umount ", self.log.read_text()) + + def test_external_utility_failures_leave_no_published_token(self): + for command in ("cp", "mv", "chmod", "od", "awk"): + with self.subTest(command=command): + self.run_script(ok=False, FAIL_COMMAND=command) + self.assert_empty() + self.run_script(ok=False, PARTIAL_COPY="yes") + self.assert_empty() + + def test_hardlinked_token_and_config_are_rejected(self): + os.link(self.media / TOKEN, self.root / "linked-token") + self.run_script(ok=False) + self.assert_empty() + + def test_hidden_stale_files_removed_but_directories_preserved(self): + (self.tokens / ".old").write_bytes(b"SYNTHETIC-STALE") + self.run_script(MEDIA="none") + self.assert_empty() + (self.tokens / ".unexpected").mkdir() + self.run_script(ok=False) + self.assertTrue((self.tokens / ".unexpected").is_dir()) + + def test_interrupted_staging_is_cleaned_before_reuse(self): + stage = self.runtime / ".stage" + stage.mkdir(mode=0o700) + (stage / "token").write_bytes(b"SYNTHETIC-PARTIAL") + conf_stage = self.conf.parent / ".cozystack-vgpu-license" + conf_stage.mkdir(mode=0o700) + (conf_stage / "config").write_bytes(b"FeatureType=bad\n") + self.run_script() + self.assertFalse(stage.exists()) + self.assertFalse(conf_stage.exists()) + self.assertEqual((self.tokens / TOKEN).read_bytes(), (self.media / TOKEN).read_bytes()) + + def test_unsafe_staging_and_configuration_leave_no_token(self): + cases = ("stage-symlink", "stage-permissions", "stage-directory", "stage-unknown", + "config-hardlink", "config-writable", "parent-symlink") + for case in cases: + with self.subTest(case=case): + self.conf.unlink(missing_ok=True) + stage = self.runtime / ".stage" + if stage.is_symlink(): stage.unlink() + elif stage.exists(): shutil.rmtree(stage) + if case == "stage-symlink": stage.symlink_to(self.default) + elif case.startswith("stage-"): + stage.mkdir(mode=0o700) + if case == "stage-permissions": stage.chmod(0o755) + if case == "stage-directory": (stage / "token").mkdir() + if case == "stage-unknown": (stage / "unexpected").touch() + elif case == "config-hardlink": os.link(self.outside, self.conf) + elif case == "config-writable": + self.conf.write_bytes(b"FeatureType=1\n") + self.conf.chmod(0o666) + elif case == "parent-symlink": + self.conf.parent.rename(self.root / "nvidia-real") + self.conf.parent.symlink_to(self.root / "nvidia-real") + self.run_script(ok=False) + self.assert_empty() + + def test_unrelated_config_mode_is_not_widened(self): + self.conf.write_text("VendorSetting=yes\n") + self.conf.chmod(0o600) + self.run_script() + self.assertEqual(stat.S_IMODE(self.conf.stat().st_mode), 0o600) + self.assertIn("VendorSetting=yes", self.conf.read_text()) + + def test_config_symlink_rejected(self): + self.conf.symlink_to(self.outside) + self.run_script(ok=False) + + def test_workflow_runs_both_lanes_and_windows_engines(self): + workflow = (ROOT / ".github/workflows/vgpu-guest.yml").read_text() + linux = workflow.split("\n linux:\n", 1)[1].split("\n windows:\n", 1)[0] + windows = workflow.split("\n windows:\n", 1)[1] + self.assertIn("python3 -I hack/test_vgpu_guest.py", linux) + self.assertIn("shell: [powershell, pwsh]", windows) + self.assertIn("shell: ${{ matrix.shell }}", windows) + self.assertIn("$config.Run.Path = './hack/Test-VgpuGuest.Tests.ps1'", windows) + self.assertIn("$config.Run.Throw = $true", windows) + self.assertIn("$result.TotalCount -eq 0", windows) + for lane in (linux, windows): + self.assertIn("fail-fast: false", lane) + for path in ("static/examples/vgpu-guest/**", "hack/test_vgpu_guest.py", + "hack/Test-VgpuGuest.Tests.ps1", ".github/workflows/vgpu-guest.yml"): + self.assertEqual(workflow.count(f" - '{path}'"), 2, path) + + def test_empty_test_selection_is_not_success(self): + result = subprocess.run([sys.executable, "-I", __file__, "-k", "NO_MATCHING_VGPU_TEST"], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_test_root_must_be_explicit_and_match_environment(self): + result = subprocess.run(["sh", str(SCRIPT), "--test-root", "/"], + env=self.env, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.log.exists()) + + def test_client_only_dependency_not_boot_target(self): + unit = (ASSETS / "cozystack-vgpu-license.service").read_text() + dropin = (ASSETS / "nvidia-gridd.service.d/cozystack-vgpu-license.conf").read_text() + self.assertIn("Type=oneshot", unit) + self.assertNotIn("RemainAfterExit=", unit) + self.assertIn("Before=nvidia-gridd.service", unit) + self.assertIn("ExecStart=/usr/local/sbin/cozystack-vgpu-license", unit) + self.assertIn("Requires=cozystack-vgpu-license.service", dropin) + self.assertIn("After=cozystack-vgpu-license.service", dropin) + self.assertNotIn("WantedBy=", unit) + self.assertNotIn("RequiredBy=", unit) + self.assertNotIn("FailureAction=", unit) + self.assertNotIn("OnFailure=", unit) + + +@unittest.skipUnless(sys.platform.startswith("linux"), "Requires Linux util-linux and e2fsprogs; no mounts or block devices") +class LinuxProbeFormat(unittest.TestCase): + def test_export_tag_filter_still_includes_device_name(self): + for tool in ("mkfs.ext4", "blkid"): + self.assertIsNotNone(shutil.which(tool), f"Linux CI prerequisite missing: {tool}") + with tempfile.TemporaryDirectory(prefix="vgpu-probe-") as temporary: + image = Path(temporary) / "synthetic-license.img" + image.write_bytes(bytes(4 * 1024 * 1024)) + subprocess.run(["mkfs.ext4", "-F", "-L", "GPULICENSE", str(image)], + check=True, capture_output=True) + result = subprocess.run(["blkid", "--probe", "--output", "export", "--match-tag", "LABEL", "--", str(image)], + check=True, capture_output=True, text=True) + self.assertEqual(result.stdout, f"DEVNAME={image}\nLABEL=GPULICENSE\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/static/examples/vgpu-guest/v1/SHA256SUMS b/static/examples/vgpu-guest/v1/SHA256SUMS new file mode 100644 index 000000000..deee74827 --- /dev/null +++ b/static/examples/vgpu-guest/v1/SHA256SUMS @@ -0,0 +1,4 @@ +bafc04a10e86adec242703e4265488f69d277041d96badd6eae4e4cfc2eb7b33 linux/cozystack-vgpu-license.service +b54395362154c403104cb6f040df4e09e082d8e82a739b81f0a6d73ca9fb4b72 linux/cozystack-vgpu-license.sh +ad155a6acb51665394a796eb445c4521cfa91529b6a1afa4b20969ef1727f1df linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf +ba2f1185db5668a461dd49ff5ae896aeb2b5576ff5d4ea73702919ca98590215 windows/Set-CozystackVgpuLicense.ps1 diff --git a/static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.service b/static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.service new file mode 100644 index 000000000..0e2799519 --- /dev/null +++ b/static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.service @@ -0,0 +1,8 @@ +[Unit] +Description=Prepare the platform vGPU license token +Before=nvidia-gridd.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/cozystack-vgpu-license +UMask=0022 diff --git a/static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.sh b/static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.sh new file mode 100755 index 000000000..13a8702f5 --- /dev/null +++ b/static/examples/vgpu-guest/v1/linux/cozystack-vgpu-license.sh @@ -0,0 +1,268 @@ +#!/bin/sh +set +x +umask 077 +LC_ALL=C +export LC_ALL +root= +if [ "$#" != 0 ]; then + if [ "$#" != 2 ] || [ "$1" != --test-root ] || [ "$2" != "${COZYSTACK_VGPU_TEST_ROOT:-}" ]; then + echo 'Invalid sandbox invocation' >&2 + exit 1 + fi + case "$2" in + /|*..*|*[!A-Za-z0-9_./-]*|'') echo 'Invalid sandbox invocation' >&2; exit 1 ;; + /*) root=$2 ;; + *) echo 'Invalid sandbox invocation' >&2; exit 1 ;; + esac + [ -d "$root" ] && [ ! -L "$root" ] || exit 1 +else + PATH=/usr/sbin:/usr/bin:/sbin:/bin + export PATH +fi +uid=$(id -u) || exit 1 +if [ -z "$root" ] && [ "$uid" != 0 ]; then + echo 'Root is required' >&2 + exit 1 +fi +runtime=$root/run/cozystack-vgpu-license +tokens=$runtime/token +media=$runtime/media +stage=$runtime/.stage +config=$root/etc/nvidia/gridd.conf +config_stage=$root/etc/nvidia/.cozystack-vgpu-license +label_link=$root/dev/disk/by-label/GPULICENSE +mounted=false +cleanup_ready=false +stage_ready=false +config_stage_ready=false + +fail() { + echo "$1" >&2 + exit 1 +} + +directory() { + if [ "$1" != "$root" ] && [ "$1" != / ]; then + case "${1%/*}" in + '') directory / 0755 || return 1 ;; + *) directory "${1%/*}" 0755 || return 1 ;; + esac + fi + if [ ! -e "$1" ] && [ ! -L "$1" ]; then + mkdir -m "$2" "$1" 2>/dev/null || return 1 + fi + found=$(find "$1" -prune -type d -user "$uid" ! -perm -0020 ! -perm -0002 -print 2>/dev/null) || return 1 + [ "$found" = "$1" ] || return 1 + if [ "$2" = 0700 ]; then + found=$(find "$1" -prune -perm 0700 -print 2>/dev/null) || return 1 + [ "$found" = "$1" ] || return 1 + fi +} + +regular() { + found=$(find "$1" -prune -type f -links 1 -print 2>/dev/null) || return 1 + [ "$found" = "$1" ] +} + +trusted_file() { + regular "$1" || return 1 + found=$(find "$1" -prune -user "$uid" ! -perm -0020 ! -perm -0002 -print 2>/dev/null) || return 1 + [ "$found" = "$1" ] +} + +mount_id() { + identity=$(findmnt --noheadings --raw --output ID --target "$1" 2>/dev/null) || return 1 + case "$identity" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' "$identity" +} + +clear_tokens() { + token_mount=$(mount_id "$tokens") || return 1 + for entry in "$tokens"/* "$tokens"/.[!.]* "$tokens"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + if [ ! -L "$entry" ]; then + [ ! -d "$entry" ] || return 1 + entry_mount=$(mount_id "$entry") || return 1 + [ "$entry_mount" = "$token_mount" ] || return 1 + fi + done + for entry in "$tokens"/* "$tokens"/.[!.]* "$tokens"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + rm "$entry" 2>/dev/null || return 1 + done +} + +clear_stage() { + parent_mount=$(mount_id "$1") || return 1 + for entry in "$1"/* "$1"/.[!.]* "$1"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + case "${entry##*/}" in + token|config|bytes|inventory) ;; + *) return 1 ;; + esac + regular "$entry" || return 1 + entry_mount=$(mount_id "$entry") || return 1 + [ "$entry_mount" = "$parent_mount" ] || return 1 + done + for entry in "$1"/token "$1"/config "$1"/bytes "$1"/inventory; do + [ -e "$entry" ] || continue + rm "$entry" 2>/dev/null || return 1 + done +} + +finish() { + status=$? + trap - 0 + if "$mounted"; then + if ! umount -- "$media" >/dev/null 2>&1; then + echo 'vGPU license media unmount failed' >&2 + status=1 + fi + fi + if "$stage_ready"; then + if ! clear_stage "$stage" || ! rmdir "$stage" 2>/dev/null; then + echo 'vGPU token staging cleanup failed' >&2 + status=1 + fi + fi + if "$config_stage_ready"; then + if ! clear_stage "$config_stage" || ! rmdir "$config_stage" 2>/dev/null; then + echo 'vGPU config staging cleanup failed' >&2 + status=1 + fi + fi + if [ "$status" != 0 ] && "$cleanup_ready"; then + clear_tokens || echo 'vGPU managed token cleanup failed' >&2 + fi + exit "$status" +} +trap finish 0 +trap 'exit 1' HUP INT TERM + +text_file() { + od -A n -v -t u1 "$1" > "$config_stage/bytes" 2>/dev/null || return 1 + awk -v ascii="$2" ' + { for (i = 1; i <= NF; i++) { + if (($i < 32 && $i != 9 && $i != 10 && $i != 13) || $i == 127 || (ascii && $i > 126)) bad = 1 + if (previous == 13 && $i != 10) bad = 1 + previous = $i + } } + END { exit (bad || previous == 13) } + ' "$config_stage/bytes" 2>/dev/null +} + +merge_config() { + source=/dev/null + if [ -e "$config" ] || [ -L "$config" ]; then + trusted_file "$config" || return 1 + text_file "$config" 0 || return 1 + source=$config + cp -p "$config" "$config_stage/config" 2>/dev/null || return 1 + else + : > "$config_stage/config" || return 1 + chmod 0644 "$config_stage/config" 2>/dev/null || return 1 + fi + awk -v path="$tokens" -v feature="$1" ' + /^[ \t]*ClientConfigTokenPath[ \t]*=/ { next } + feature != "" && /^[ \t]*FeatureType[ \t]*=/ { next } + { print } + END { + print "ClientConfigTokenPath=" path + if (feature != "") print "FeatureType=" feature + } + ' "$source" > "$config_stage/config" 2>/dev/null || return 1 + trusted_file "$config_stage/config" || return 1 + if [ -e "$config" ] || [ -L "$config" ]; then + trusted_file "$config" || return 1 + fi + mv "$config_stage/config" "$config" 2>/dev/null +} + +block_device() { + if [ -n "$root" ]; then + case "$1" in "$root"/*) [ -f "$1" ] ;; *) return 1 ;; esac + else + [ -b "$1" ] + fi +} + +for path in "$runtime" "$tokens" "$media"; do + directory "$path" 0755 || fail 'Unsafe vGPU runtime directory' +done +cleanup_ready=true +mounts=$(findmnt --kernel --list --noheadings --raw --output TARGET 2>/dev/null) || fail 'vGPU mount inventory failed' +newline=' +' +for path in "$media" "$stage" "$config_stage"; do + case "$newline$mounts$newline" in *"$newline$path$newline"*) fail 'vGPU mountpoint is already in use' ;; esac +done +clear_tokens || fail 'vGPU managed token cleanup failed' +for path in "$tokens" "$runtime"; do + filesystem=$(findmnt --noheadings --raw --output FSTYPE --target "$path" 2>/dev/null) || fail 'vGPU filesystem discovery failed' + case "$filesystem" in tmpfs|ramfs) ;; *) fail 'vGPU token directory requires a memory filesystem' ;; esac +done +directory "$stage" 0700 || fail 'Unsafe vGPU token staging directory' +stage_ready=true +stage_mount=$(mount_id "$stage") || fail 'vGPU token staging discovery failed' +token_mount=$(mount_id "$tokens") || fail 'vGPU token filesystem discovery failed' +[ "$stage_mount" = "$token_mount" ] || fail 'vGPU token staging must share the token mount' +clear_stage "$stage" || fail 'Unsafe vGPU token staging contents' +directory "$config_stage" 0700 || fail 'Unsafe vGPU configuration directory' +config_stage_ready=true +config_mount=$(mount_id "${config%/*}") || fail 'vGPU config mount discovery failed' +staging_mount=$(mount_id "$config_stage") || fail 'vGPU config staging discovery failed' +[ "$config_mount" = "$staging_mount" ] || fail 'vGPU config staging must share the config mount' +clear_stage "$config_stage" || fail 'Unsafe vGPU configuration staging contents' +merge_config '' || fail 'vGPU license configuration preparation failed' + +udevadm settle --timeout=10 >/dev/null 2>&1 || fail 'vGPU device discovery did not settle' +lsblk --list --noheadings --paths --raw --output NAME,LABEL > "$stage/inventory" 2>/dev/null || fail 'vGPU block inventory failed' +device=$(awk ' + NF > 2 || $1 !~ /^\// { bad = 1 } + $2 == "GPULICENSE" { count++; device = $1 } + END { if (bad || count > 1) exit 1; if (count == 1) print device } +' "$stage/inventory" 2>/dev/null) || fail 'Ambiguous or invalid vGPU license inventory' +if [ -e "$label_link" ] || [ -L "$label_link" ]; then + block_device "$label_link" || fail 'Stale vGPU license label link' + [ -n "$device" ] || fail 'vGPU license label missing from inventory' + link_id=$(lsblk --nodeps --noheadings --output MAJ:MIN -- "$label_link" 2>/dev/null) || fail 'vGPU label identity discovery failed' + device_id=$(lsblk --nodeps --noheadings --output MAJ:MIN -- "$device" 2>/dev/null) || fail 'vGPU device identity discovery failed' + [ -n "$link_id" ] || fail 'vGPU license label identity mismatch' + [ "$link_id" = "$device_id" ] || fail 'vGPU license label identity mismatch' +fi +[ -n "$device" ] || exit 0 +case "$device" in *[!A-Za-z0-9_./-]*) fail 'Unsupported vGPU license device path' ;; esac +block_device "$device" || fail 'vGPU license candidate is not a block device' +label=$(blkid --probe --output export --match-tag LABEL -- "$device" 2>/dev/null) || fail 'vGPU license media probe failed' +[ "$label" = "DEVNAME=$device${newline}LABEL=GPULICENSE" ] || fail 'vGPU license candidate label mismatch' +mount --options ro,nodev,nosuid,noexec -- "$device" "$media" >/dev/null 2>&1 || fail 'vGPU license media mount failed' +mounted=true +device_id=$(lsblk --nodeps --noheadings --raw --output MAJ:MIN -- "$device" 2>/dev/null) || fail 'vGPU device identity discovery failed' +mounted_id=$(findmnt --noheadings --raw --output MAJ:MIN --mountpoint "$media" 2>/dev/null) || fail 'vGPU license mount verification failed' +[ -n "$device_id" ] || fail 'vGPU license mount source mismatch' +[ "$device_id" = "$mounted_id" ] || fail 'vGPU license mount source mismatch' +options=$(findmnt --noheadings --raw --output OPTIONS --mountpoint "$media" 2>/dev/null) || fail 'vGPU license mount verification failed' +for option in ro nodev nosuid noexec; do + case ",$options," in *",$option,"*) ;; *) fail 'Unsafe vGPU license mount options' ;; esac +done +regular "$media/client_configuration_token.tok" || fail 'Invalid vGPU license token file' +[ -s "$media/client_configuration_token.tok" ] || fail 'Invalid vGPU license token file' +regular "$media/gridd.conf" || fail 'Invalid vGPU license media configuration' +text_file "$media/gridd.conf" 1 || fail 'Invalid vGPU license media configuration' +feature=$(awk ' + { sub(/\r$/, ""); if ($0 ~ /\r/) bad = 1 } + /^[ \t]*FeatureType[ \t]*=/ { + count++; sub(/^[ \t]*FeatureType[ \t]*=[ \t]*/, ""); sub(/[ \t]*$/, "") + if ($0 != "1" && $0 != "4") bad = 1 + feature = $0 + } + END { if (bad || count != 1) exit 1; print feature } +' "$media/gridd.conf" 2>/dev/null) || fail 'Invalid vGPU license FeatureType' +cp "$media/client_configuration_token.tok" "$stage/token" 2>/dev/null || fail 'vGPU license token copy failed' +regular "$stage/token" || fail 'Invalid staged vGPU token' +[ -s "$stage/token" ] || fail 'Invalid staged vGPU token' +chmod 0744 "$stage/token" 2>/dev/null || fail 'vGPU license token mode failed' +merge_config "$feature" || fail 'vGPU license configuration merge failed' +[ ! -e "$tokens/client_configuration_token.tok" ] || fail 'Unexpected vGPU token destination' +[ ! -L "$tokens/client_configuration_token.tok" ] || fail 'Unexpected vGPU token destination' +mv "$stage/token" "$tokens/client_configuration_token.tok" 2>/dev/null || fail 'vGPU license token installation failed' diff --git a/static/examples/vgpu-guest/v1/linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf b/static/examples/vgpu-guest/v1/linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf new file mode 100644 index 000000000..4ab00c780 --- /dev/null +++ b/static/examples/vgpu-guest/v1/linux/nvidia-gridd.service.d/cozystack-vgpu-license.conf @@ -0,0 +1,3 @@ +[Unit] +Requires=cozystack-vgpu-license.service +After=cozystack-vgpu-license.service diff --git a/static/examples/vgpu-guest/v1/windows/Set-CozystackVgpuLicense.ps1 b/static/examples/vgpu-guest/v1/windows/Set-CozystackVgpuLicense.ps1 new file mode 100644 index 000000000..310eb713c --- /dev/null +++ b/static/examples/vgpu-guest/v1/windows/Set-CozystackVgpuLicense.ps1 @@ -0,0 +1,73 @@ +[CmdletBinding()] +param([string]$TestRoot) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +if ($PSBoundParameters.ContainsKey('TestRoot')) { + if (-not $TestRoot -or $TestRoot -ne $env:COZYSTACK_VGPU_TEST_ROOT -or + -not (Test-Path -LiteralPath $TestRoot -PathType Container) -or + [IO.Path]::GetPathRoot($TestRoot) -eq $TestRoot) { + throw 'Invalid sandbox invocation' + } + $drive = $TestRoot +} else { + $drive = $env:SystemDrive + '\' +} +$folder = Join-Path $drive 'Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken' + +function Assert-DirectoryPath([string]$Path) { + $parent = Split-Path -Parent $Path + if ($parent -and $parent -ne $Path) { Assert-DirectoryPath $parent } + if (Test-Path -LiteralPath $Path) { + $item = Get-Item -LiteralPath $Path -Force + if (-not $item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw 'Unsafe token directory' + } + } +} + +function Clear-TokenFolder { + foreach ($item in @(Get-ChildItem -LiteralPath $folder -Force)) { + if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw 'Unexpected token folder entry' + } + Remove-Item -LiteralPath $item.FullName -Force -ErrorAction Stop + } +} + +try { + Assert-DirectoryPath $folder + if (-not (Test-Path -LiteralPath $folder)) { + New-Item -ItemType Directory -Path $folder -Force | Out-Null + } + Clear-TokenFolder +} catch { + throw 'vGPU license token cleanup failed' +} + +$mediaFailed = $false +try { + $volumes = @(Get-Volume -ErrorAction Stop | Where-Object { $_.FileSystemLabel -ceq 'GPULICENSE' }) + if ($volumes.Count -gt 1) { throw 'Ambiguous media' } + if ($volumes.Count -eq 1) { + if (-not $volumes[0].Path) { throw 'Missing volume path' } + $source = Join-Path $volumes[0].Path 'client_configuration_token.tok' + $item = Get-Item -LiteralPath $source -Force -ErrorAction Stop + if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $item.Length -eq 0) { + throw 'Invalid token file' + } + $temporary = Join-Path $folder '.cozystack-token.tmp' + Copy-Item -LiteralPath $source -Destination $temporary -ErrorAction Stop + Move-Item -LiteralPath $temporary -Destination (Join-Path $folder 'client_configuration_token.tok') -ErrorAction Stop + } +} catch { + $mediaFailed = $true + try { Clear-TokenFolder } catch { throw 'vGPU license token cleanup failed' } +} + +try { + Restart-Service -Name NvDisplayContainer -ErrorAction Stop +} catch { + throw 'vGPU license client restart failed' +} +if ($mediaFailed) { throw 'vGPU license media preparation failed' }