diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs index 7a01210243..f0bef411fb 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs @@ -5,6 +5,7 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; + using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -45,6 +46,44 @@ public void SpecCpuWorkloadProfileParametersAreInlinedCorrectly(string profile) } } + [Test] + [TestCase("PERF-SPECCPU-INTRATE.json", PlatformID.Unix, "intrate", null)] + [TestCase("PERF-SPECCPU-INTRATE.json", PlatformID.Unix, "intrate", "2,4,8,10")] + [TestCase("PERF-SPECCPU-INTRATE.json", PlatformID.Win32NT, "505 525 541 548 557", null)] + [TestCase("PERF-SPECCPU-FPRATE.json", PlatformID.Unix, "fprate", null)] + [TestCase("PERF-SPECCPU-FPRATE.json", PlatformID.Unix, "fprate", "2,4,8,10")] + [TestCase("PERF-SPECCPU-FPRATE.json", PlatformID.Win32NT, "503 508 510 519 544 549 554", null)] + public async Task SpecCpuRateProfilesRunTheFullSuiteWithOptionalNumaProcessorAffinity( + string profileName, + PlatformID platform, + string expectedBenchmarks, + string numaProcessorAffinity) + { + this.mockFixture.Setup(platform); + + ExecutionProfile profile = await ExecutionProfile.ReadProfileAsync(Path.Combine(MockFixture.TestResourcesDirectory, "profiles", profileName)); + if (numaProcessorAffinity != null) + { + profile.Parameters["NumaProcessorAffinity"] = numaProcessorAffinity; + } + + profile.Parameters["CopyCount"] = 4; + + Assert.AreEqual(1, profile.Actions.Count, "A SPEC CPU rate suite must run as one action, not separate benchmarks."); + ExecutionProfileElement action = profile.Actions.Single(); + Assert.AreEqual("SpecCpuExecutor", action.Type); + Assert.AreEqual("ExecuteSPECBenchmark", action.Parameters["Scenario"]); + Assert.AreEqual("$.Parameters.Benchmarks", action.Parameters["Benchmarks"], "The suite action must use the platform-specific benchmark selection."); + Assert.IsFalse(profile.Actions.Any(item => item.Parameters["Scenario"].ToString().StartsWith("benchmark_", StringComparison.OrdinalIgnoreCase))); + + profile.Inline(); + await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, action.Parameters); + + Assert.AreEqual(expectedBenchmarks, action.Parameters["Benchmarks"]); + Assert.AreEqual(numaProcessorAffinity ?? string.Empty, action.Parameters["NumaProcessorAffinity"]); + Assert.AreEqual("4", action.Parameters["CopyCount"].ToString()); + } + [Test] [TestCase("PERF-SPECCPU2026-FPRATE.json", Architecture.X64)] [TestCase("PERF-SPECCPU2026-FPRATE.json", Architecture.Arm64)] diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs index f26f41d8e8..6139f41fd3 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs @@ -10,6 +10,7 @@ namespace VirtualClient.Actions using System.IO.Abstractions; using System.Linq; using System.Reflection; + using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -589,10 +590,249 @@ public async Task SpecCpu2017ExecutorDoesNotApplyGcc15WorkaroundWhenGccVersionIs Assert.IsFalse(writtenConfigText.Contains("$Gcc10Workaround$"), "Placeholder '$Gcc10Workaround$' should be replaced."); } - private void SetupLinux() + [TestCase("intrate", "0-3", "$SPECCOPYNUM", null)] + [TestCase("intrate", "0,1,48,49", "$BIND", "bind = 0,1,48,49")] + [TestCase("intrate", "2,4,8,10", "$BIND", "bind = 2,4,8,10")] + [TestCase("fprate", "2,4,8,10", "$BIND", "bind = 2,4,8,10")] + public async Task SpecCpu2017ExecutorPinsRateCopies(string profile, string cpuIds, string copyCpu, string expectedBind) + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(cpuIds); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.SpecProfile)] = profile; + this.mockFixture.Parameters[nameof(SpecCpuExecutor.Benchmarks)] = profile; + + string pinnedConfig = null; + string pinnedPath = null; + this.mockFixture.File.Setup(f => f.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + pinnedPath = path; + pinnedConfig = content; + }) + .Returns(Task.CompletedTask); + + List commands = new List(); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + commands.Add($"{exe} {arguments}"); + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + StringAssert.Contains("vc-linux-x64-2017-affinity-", Path.GetFileName(pinnedPath)); + StringAssert.Contains($"submit = numactl --localalloc --physcpubind={copyCpu} -- $command", pinnedConfig); + if (expectedBind != null) + { + StringAssert.Contains(expectedBind, pinnedConfig); + } + + Assert.IsTrue(commands.Any(command => command == "numactl --version")); + Assert.IsTrue(commands.Any(command => command == $"numactl --localalloc -C {cpuIds} true")); + Assert.IsTrue(commands.Any(command => command.Contains($"--config {Path.GetFileName(pinnedPath)}") && command.Contains("--copies 4"))); + } + + [TestCase("", false, Architecture.X64)] + [TestCase("0-3", true, Architecture.X64)] + [TestCase("0-3", true, Architecture.Arm64)] + public async Task SpecCpu2017ExecutorPreservesSuiteTelemetryAndMetadataWithOptionalAffinity(string numaProcessorAffinity, bool pinned, Architecture architecture) + { + this.SetupLinux(architecture); + this.SetupInstalledAffinityRun(numaProcessorAffinity); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.Scenario)] = "ExecuteSPECBenchmark"; + string topology = "available: 2 nodes (0-1)\nnode 0 cpus: 0 1\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB\nnode 1 cpus: 2 3\nnode 1 size: 16000 MB\nnode 1 free: 11000 MB"; + + string configurationFile = null; + this.mockFixture.File.Setup(f => f.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => configurationFile = Path.GetFileName(path)) + .Returns(Task.CompletedTask); + + List commands = new List(); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + commands.Add($"{exe} {arguments}"); + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder(exe == "numactl" && arguments == "--hardware" ? topology : string.Empty)), + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + string expectedConfig = pinned ? configurationFile : $"vc-linux-{(architecture == Architecture.Arm64 ? "arm64" : "x64")}-2017.cfg"; + Assert.AreEqual(pinned, commands.Any(command => command == "numactl --version")); + Assert.AreEqual(pinned, commands.Any(command => command == "numactl --hardware")); + Assert.AreEqual(1, commands.Count(command => command.StartsWith("bash runspeccpu.sh", StringComparison.Ordinal))); + Assert.IsTrue(commands.Any(command => command.Contains($"--config {expectedConfig}") && command.EndsWith("--reportable intrate\""))); + + var topologyEvents = this.mockFixture.Logger.Where(entry => entry.Item2.Name.EndsWith(".NumactlTopology", StringComparison.Ordinal)).ToList(); + Assert.AreEqual(pinned ? 1 : 0, topologyEvents.Count); + if (pinned) + { + EventContext topologyContext = (EventContext)topologyEvents.Single().Item3; + Assert.AreEqual("0,1,2,3", topologyContext.Properties["cpuIds"]); + Assert.AreEqual(true, topologyContext.Properties["available"]); + Assert.AreEqual(topology, topologyContext.Properties["topology"]); + } + + EventContext runContext = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".RunStarted", StringComparison.Ordinal)).Item3; + Assert.AreEqual("ExecuteSPECBenchmark", runContext.Properties["scenario"]); + Assert.AreEqual(pinned, runContext.Properties["pinningEnabled"]); + Assert.AreEqual(expectedConfig, runContext.Properties["configurationFile"]); + + EventContext metricsContext = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".MetricsCaptured", StringComparison.Ordinal)).Item3; + IDictionary metadata = (IDictionary)metricsContext.Properties["metadata_scenario"]; + Assert.AreEqual("SPECcpu/intrate", metadata["toolName"]); + StringAssert.Contains($"--config {expectedConfig}", metadata["toolArguments"].ToString()); + + var numaMetrics = this.mockFixture.Logger.Where(entry => entry.Item2.Name == "SPECcpu.ScenarioResult").ToList(); + Assert.AreEqual(pinned ? 6 : 0, numaMetrics.Count); + if (pinned) + { + EventContext cpuMetric = (EventContext)numaMetrics.Single(entry => ((EventContext)entry.Item3).Properties["metricName"].Equals("node-cpus-0")).Item3; + Assert.AreEqual(2, cpuMetric.Properties["metricValue"]); + Assert.AreEqual("count", cpuMetric.Properties["metricUnit"]); + Assert.AreEqual("NUMA-diagnostic", cpuMetric.Properties["metricCategorization"]); + Assert.AreEqual("ExecuteSPECBenchmark", cpuMetric.Properties["scenarioName"]); + IDictionary cpuMetadata = (IDictionary)cpuMetric.Properties["metadata_metrics"]; + Assert.AreEqual("0,1", cpuMetadata["cpuIds"]); + Assert.AreEqual("0,1", cpuMetadata["selectedCpuIds"]); + Assert.AreEqual("0,1,2,3", cpuMetadata["numaProcessorAffinity"]); + Assert.AreEqual("physcpubind", cpuMetadata["processorBinding"]); + Assert.AreEqual("localalloc", cpuMetadata["numaMemoryPolicy"]); + Assert.AreEqual("5", cpuMetric.Properties["metricVerbosity"]); + EventContext secondNodeCpuMetric = (EventContext)numaMetrics.Single(entry => ((EventContext)entry.Item3).Properties["metricName"].Equals("node-cpus-1")).Item3; + IDictionary secondNodeMetadata = (IDictionary)secondNodeCpuMetric.Properties["metadata_metrics"]; + Assert.AreEqual("2,3", secondNodeMetadata["cpuIds"]); + Assert.AreEqual("2,3", secondNodeMetadata["selectedCpuIds"]); + + EventContext sizeMetric = (EventContext)numaMetrics.Single(entry => ((EventContext)entry.Item3).Properties["metricName"].Equals("node-size-0")).Item3; + Assert.AreEqual(16000, sizeMetric.Properties["metricValue"]); + Assert.AreEqual("MB", sizeMetric.Properties["metricUnit"]); + EventContext freeMetric = (EventContext)numaMetrics.Single(entry => ((EventContext)entry.Item3).Properties["metricName"].Equals("node-free-1")).Item3; + Assert.AreEqual(11000, freeMetric.Properties["metricValue"]); + Assert.AreEqual(cpuMetadata["topologyCapturedAt"], ((IDictionary)freeMetric.Properties["metadata_metrics"])["topologyCapturedAt"]); + Assert.Less(this.mockFixture.Logger.IndexOf(numaMetrics.Last()), this.mockFixture.Logger.FindIndex(entry => entry.Item2.Name.EndsWith(".MetricsCaptured", StringComparison.Ordinal))); + } + } + + [TestCase("", 1)] + [TestCase("available: 2 nodes (0-1)\nnode 0 cpus: 0 1\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB\nnode 1 cpus: 2 3\nnode 1 size: 16000 MB", 0)] + [TestCase("available: 1 nodes (0)\nnode 0 cpus: 8 9 10 11\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB", 0)] + public async Task SpecCpu2017ExecutorReportsUnavailableTopologyWithoutBlockingTheRun(string topology, int probeExitCode) + { + this.SetupLinux(Architecture.Arm64); + this.SetupInstalledAffinityRun("0-3"); + bool ranSuite = false; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + ranSuite |= exe == "bash"; + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder(exe == "numactl" && arguments == "--hardware" ? topology : string.Empty)), + ExitCode = exe == "numactl" && arguments == "--hardware" ? probeExitCode : 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + EventContext topologyContext = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".NumactlTopology", StringComparison.Ordinal)).Item3; + Assert.AreEqual(false, topologyContext.Properties["available"]); + Assert.IsFalse(this.mockFixture.Logger.Any(entry => entry.Item2.Name == "SPECcpu.ScenarioResult")); + Assert.IsTrue(ranSuite); + } + + [TestCase("0-2")] + [TestCase("0,invalid,2,3")] + public void SpecCpu2017ExecutorRejectsInvalidNumaProcessorAffinity(string cpuIds) + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(cpuIds); + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsAffinityOnSpeedRuns() + { + this.SetupLinux(); + this.SetupInstalledAffinityRun("0-3"); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.SpecProfile)] = "intspeed"; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [TestCase("--version", typeof(DependencyException))] + [TestCase("--localalloc", typeof(ArgumentException))] + public void SpecCpu2017ExecutorValidatesNumactl(string failingCommand, Type expectedException) + { + this.SetupLinux(); + this.SetupInstalledAffinityRun("0-3"); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + ExitCode = exe == "numactl" && arguments.StartsWith(failingCommand) ? 1 : 0, + OnStart = () => true, + OnHasExited = () => true + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(expectedException, async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsAffinityOnWindows() + { + this.SetupWindows(); + this.SetupInstalledAffinityRun("0-3"); + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + private void SetupInstalledAffinityRun(string cpuIds) + { + State state = new State(new Dictionary { ["SpecCpuInitialized"] = true }); + this.mockFixture.StateManager.OnGetState().ReturnsAsync(JObject.Parse(state.ToJson())); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.NumaProcessorAffinity)] = cpuIds; + } + + private void SetupLinux(Architecture architecture = Architecture.X64) { this.mockFixture = new MockFixture(); - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(PlatformID.Unix, architecture); this.mockPackage = new DependencyPath("SPECcpu", this.mockFixture.PlatformSpecifics.GetPackagePath("speccpu", "1.1.8")); this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPackage); diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs index acfa03c6dc..2fa904b485 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs @@ -1,4 +1,5 @@ -// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + // Licensed under the MIT License. namespace VirtualClient.Actions diff --git a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs index cc73b22d66..4a7ce25aa9 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs @@ -5,18 +5,24 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; + using System.ComponentModel; + using System.Globalization; using System.IO; using System.IO.Abstractions; using System.Linq; using System.Runtime.InteropServices; + using System.Security.Cryptography; using System.Text; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using global::VirtualClient.Common; using global::VirtualClient.Common.Extensions; + using global::VirtualClient.Common.ProcessAffinity; using global::VirtualClient.Common.Telemetry; using global::VirtualClient.Contracts; using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; using VirtualClient.Contracts.Metadata; /// @@ -37,6 +43,7 @@ public class SpecCpuExecutor : VirtualClientComponent private IPackageManager packageManager; private IStateManager stateManager; private ISystemManagement systemManager; + private NumaTopologySnapshot numaTopology; private string tuning; /// @@ -153,8 +160,20 @@ public int CopyCount } } + /// + /// Optional list of physical CPU IDs to assign to SPECrate copies, one CPU per copy. + /// + public string NumaProcessorAffinity + { + get + { + return this.Parameters.GetValue(nameof(SpecCpuExecutor.NumaProcessorAffinity), string.Empty); + } + } + /// /// A feature flag to apply. For example 'UseCsvResults' can be used to parse the CSV file results vs. the standard output. + /// The CSV results have finer-grained results with additional significant figures. /// public string FeatureFlag { @@ -216,11 +235,22 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel /// A token that can be used to cancel the operation. protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, CancellationToken cancellationToken) { + string stage = "Configuration"; + this.numaTopology = null; try { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - string commandLineArguments = this.GetCommandLineArguments(); + string configurationFile = this.GetConfigurationFileName(); + if (!string.IsNullOrWhiteSpace(this.NumaProcessorAffinity)) + { + stage = "AffinityConfiguration"; + configurationFile = await this.WritePinnedSpecCpuConfigAsync(telemetryContext, cancellationToken); + } + + this.LogSpecCpuStep(telemetryContext, "ConfigSelected", ("configurationFile", configurationFile)); + string commandLineArguments = this.GetCommandLineArguments(configurationFile); + string command; string commandArguments; @@ -235,6 +265,9 @@ protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, Cancella commandArguments = $"/c {SpecCpuExecutor.SpecCpuRunBat} {commandLineArguments}"; } + stage = "Run"; + this.LogSpecCpuStep(telemetryContext, "RunStarted", ("configurationFile", configurationFile), ("command", command), ("arguments", commandArguments)); + DateTime runStart = DateTime.UtcNow; using (IProcessProxy process = await this.ExecuteCommandAsync( command, commandArguments, @@ -246,13 +279,28 @@ protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, Cancella if (!cancellationToken.IsCancellationRequested) { await this.LogProcessDetailsAsync(process, telemetryContext, "SPECcpu", logToFile: true); + this.LogSpecCpuStep(telemetryContext, "RunCompleted", ("configurationFile", configurationFile), ("exitCode", process.ExitCode), ("durationMs", (DateTime.UtcNow - runStart).TotalMilliseconds)); process.ThrowIfWorkloadFailed(); + stage = "Metrics"; + this.MetadataContract.AddForScenario($"SPECcpu/{this.SpecProfile}", process.FullCommand(), toolVersion: null); + this.MetadataContract.Apply(telemetryContext); + await this.CaptureNumaMetricsAsync(commandLineArguments, telemetryContext, cancellationToken); await this.CaptureMetricsAsync(process, commandLineArguments, telemetryContext, cancellationToken); } } } } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + this.LogSpecCpuStep(telemetryContext, "Cancelled", ("stage", stage)); + throw; + } + catch (Exception exc) + { + this.LogSpecCpuFailure(telemetryContext, stage, exc); + throw; + } finally { if (!cancellationToken.IsCancellationRequested) @@ -269,28 +317,48 @@ protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, Cancella /// A token that can be used to cancel the operation. protected async Task InitializeSpecCpuAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None); - - if (workloadPackage == null) + string stage = "PackageLookup"; + try { - throw new DependencyException( - $"The expected package '{this.PackageName}' does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } + this.LogSpecCpuStep(telemetryContext, "PackageLookupStarted", ("packageName", this.PackageName)); + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None); - this.PackageDirectory = workloadPackage.Path; - this.ResultsDirectory = this.Combine(this.PackageDirectory, "result"); + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } - if (this.fileSystem.Directory.Exists(this.ResultsDirectory)) - { - await this.fileSystem.Directory.DeleteAsync(this.ResultsDirectory); - this.fileSystem.Directory.CreateDirectory(this.ResultsDirectory); - } + this.PackageDirectory = workloadPackage.Path; + this.ResultsDirectory = this.Combine(this.PackageDirectory, "result"); + this.LogSpecCpuStep(telemetryContext, "PackageReady", ("packageName", this.PackageName)); + + // Clean any previous results from the system. + if (this.fileSystem.Directory.Exists(this.ResultsDirectory)) + { + await this.fileSystem.Directory.DeleteAsync(this.ResultsDirectory); + this.fileSystem.Directory.CreateDirectory(this.ResultsDirectory); + } - string imageFile = this.GetIsoFilePath(workloadPackage); - telemetryContext.AddContext(nameof(imageFile), imageFile); + stage = "Setup"; + string imageFile = this.GetIsoFilePath(workloadPackage); + telemetryContext.AddContext(nameof(imageFile), imageFile); - await this.SetupSpecCpuAsync(imageFile, telemetryContext, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "SetupStarted", ("imageFile", Path.GetFileName(imageFile))); + await this.SetupSpecCpuAsync(imageFile, telemetryContext, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "SetupCompleted"); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + this.LogSpecCpuStep(telemetryContext, "Cancelled", ("stage", stage)); + throw; + } + catch (Exception exc) + { + this.LogSpecCpuFailure(telemetryContext, stage, exc); + throw; + } } /// @@ -439,11 +507,13 @@ private async Task SetupSpecCpuAsync(string isoFilePath, EventContext telemetryC SpecCpuState state = await this.stateManager.GetStateAsync(stateId, cancellationToken) ?? new SpecCpuState(); + this.LogSpecCpuStep(telemetryContext, "SetupStateRead", ("alreadyInitialized", state.SpecCpuInitialized)); if (state.SpecCpuInitialized) { return; } + this.LogSpecCpuStep(telemetryContext, "InstallationStarted"); string mountPath = this.PlatformSpecifics.Combine(this.PlatformSpecifics.GetPackagePath(), "speccpu_mount"); this.fileSystem.Directory.CreateDirectory(mountPath); @@ -458,6 +528,7 @@ private async Task SetupSpecCpuAsync(string isoFilePath, EventContext telemetryC state.SpecCpuInitialized = true; await this.stateManager.SaveStateAsync(stateId, state, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "InstallationCompleted"); } private async Task LinuxSetupAsync( @@ -557,6 +628,49 @@ private async Task WindowsSetupAsync( } } + private Task CaptureNumaMetricsAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken) + { + if (this.numaTopology == null || cancellationToken.IsCancellationRequested) + { + return Task.CompletedTask; + } + + string affinity = string.Join(",", this.numaTopology.SelectedCpus); + foreach (NumaNode node in this.numaTopology.Nodes) + { + Metric[] metrics = + [ + new Metric($"node-cpus-{node.Id}", node.Cpus.Count, "count"), + new Metric($"node-size-{node.Id}", node.SizeMb.Value, "MB"), + new Metric($"node-free-{node.Id}", node.FreeMb.Value, "MB") + ]; + + foreach (Metric metric in metrics) + { + metric.Categorization = "NUMA-diagnostic"; + metric.Verbosity = 5; + metric.Metadata["cpuIds"] = string.Join(",", node.Cpus); + metric.Metadata["selectedCpuIds"] = string.Join(",", this.numaTopology.SelectedCpus.Where(node.Cpus.Contains)); + metric.Metadata["numaProcessorAffinity"] = affinity; + metric.Metadata["processorBinding"] = "physcpubind"; + metric.Metadata["numaMemoryPolicy"] = "localalloc"; + metric.Metadata["topologyCapturedAt"] = this.numaTopology.CapturedAt.ToString("o", CultureInfo.InvariantCulture); + + this.Logger.LogMetric( + metric, + toolName: "SPECcpu", + scenarioName: this.Scenario, + this.numaTopology.CapturedAt, + this.numaTopology.CapturedAt, + commandArguments, + telemetryContext, + this.Tags); + } + } + + return Task.CompletedTask; + } + private async Task CaptureMetricsAsync( IProcessProxy process, string commandArguments, @@ -565,13 +679,6 @@ private async Task CaptureMetricsAsync( { if (!cancellationToken.IsCancellationRequested) { - this.MetadataContract.AddForScenario( - $"SPECcpu/{this.SpecProfile}", - process.FullCommand(), - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - bool useCsv = string.Equals(this.FeatureFlag, SpecCpuExecutor.FeatureFlagUseCsvResults, StringComparison.OrdinalIgnoreCase); string extension = useCsv ? "csv" : "txt"; string[] outputFiles = this.fileSystem.Directory.GetFiles( @@ -579,6 +686,7 @@ private async Task CaptureMetricsAsync( this.GetResultsFileSearchPattern(extension), SearchOption.TopDirectoryOnly); + int metricCount = 0; foreach (string file in outputFiles) { KeyValuePair results = await this.LoadResultsAsync(file, cancellationToken); @@ -588,6 +696,7 @@ private async Task CaptureMetricsAsync( IList metrics = parser.Parse(); metrics.LogConsole(this.Scenario, "SPECcpu"); + metricCount += metrics.Count; metrics.ToList().ForEach(m => m.Categorization = $"{this.SpecProfile}-{this.tuning}"); foreach (Metric metric in metrics) @@ -603,6 +712,8 @@ private async Task CaptureMetricsAsync( this.Tags); } } + + this.LogSpecCpuStep(telemetryContext, "MetricsCaptured", ("resultFileCount", outputFiles.Length), ("metricCount", metricCount), ("format", useCsv ? "csv" : "txt")); } } @@ -640,10 +751,9 @@ private async Task RequestLogUploadsAsync(CancellationToken cancellationToken) } } - private string GetCommandLineArguments() + private string GetCommandLineArguments(string configurationFile) { List suites = new List { "intrate", "intspeed", "fprate", "fpspeed" }; - string configurationFile = this.GetConfigurationFileName(); string command = $"--config {configurationFile} --iterations {this.IterationCount} --copies {this.CopyCount} --threads {this.ThreadCount} --tune {this.tuning}"; bool reportable = this.Platform == PlatformID.Unix @@ -654,6 +764,227 @@ private string GetCommandLineArguments() return $"{command} {this.Benchmarks}"; } + private async Task WritePinnedSpecCpuConfigAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.LogSpecCpuStep(telemetryContext, "AffinityValidationStarted", ("requestedCpuIds", this.NumaProcessorAffinity)); + if (this.Platform != PlatformID.Unix || + (!string.Equals(this.SpecProfile, "intrate", StringComparison.OrdinalIgnoreCase) && + !string.Equals(this.SpecProfile, "fprate", StringComparison.OrdinalIgnoreCase))) + { + throw new NotSupportedException("SPEC CPU per-copy processor affinity is supported only for Linux intrate and fprate runs."); + } + + LinuxProcessAffinityConfiguration affinityConfig = (LinuxProcessAffinityConfiguration)ProcessAffinityConfiguration.Create(this.Platform, this.NumaProcessorAffinity); + IReadOnlyList cores = affinityConfig.Cores; + if (cores.Count != this.CopyCount) + { + throw new ArgumentException($"NumaProcessorAffinity must specify exactly one distinct CPU per SPEC copy (CopyCount={this.CopyCount}, CPUs={cores.Count}).", nameof(this.NumaProcessorAffinity)); + } + + bool useCopyNumber = cores.Select((core, copy) => core == copy).All(matches => matches); + string copyCpu = useCopyNumber ? "$SPECCOPYNUM" : "$BIND"; + this.LogSpecCpuStep(telemetryContext, "AffinityValidated", ("cpuIds", string.Join(",", cores)), ("mapping", copyCpu)); + + this.LogSpecCpuStep(telemetryContext, "NumactlCheckStarted"); + try + { + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess("numactl", "--version", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + if (!cancellationToken.IsCancellationRequested && process.IsErrored()) + { + throw new DependencyException("SPEC CPU processor affinity requires a working numactl installation.", ErrorReason.WorkloadDependencyMissing); + } + + this.LogSpecCpuStep(telemetryContext, "NumactlChecked", ("exitCode", process.ExitCode)); + } + } + catch (Win32Exception exc) + { + throw new DependencyException("SPEC CPU processor affinity requires numactl to be installed.", exc, ErrorReason.WorkloadDependencyMissing); + } + + try + { + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess("numactl", "--hardware", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + + string topology = process.StandardOutput.ToString().Trim(); + IReadOnlyList nodes = null; + bool available = !process.IsErrored() && SpecCpuExecutor.TryParseNumaTopology(topology, cores, out nodes); + if (available) + { + this.numaTopology = new NumaTopologySnapshot(nodes, cores.ToArray(), DateTime.UtcNow); + } + + this.LogSpecCpuStep( + telemetryContext, + "NumactlTopology", + ("cpuIds", string.Join(",", cores)), + ("topology", topology), + ("available", available), + ("exitCode", process.ExitCode), + ("error", process.IsErrored() ? process.StandardError.ToString().Trim() : available ? string.Empty : "Unable to parse NUMA topology for the selected CPUs.")); + } + } + catch (Win32Exception exc) + { + this.LogSpecCpuStep(telemetryContext, "NumactlTopology", ("cpuIds", string.Join(",", cores)), ("available", false), ("error", exc.Message)); + } + + (string executable, string arguments) = affinityConfig.GetAffinityProcessInfo("true"); + this.LogSpecCpuStep(telemetryContext, "AffinityPreflightStarted", ("arguments", $"--localalloc {arguments}")); + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess(executable, $"--localalloc {arguments}", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + if (!cancellationToken.IsCancellationRequested && process.IsErrored()) + { + throw new ArgumentException($"NumaProcessorAffinity contains CPUs that cannot be bound with numactl --localalloc. {process.StandardError}", nameof(this.NumaProcessorAffinity)); + } + + this.LogSpecCpuStep(telemetryContext, "AffinityPreflightCompleted", ("exitCode", process.ExitCode)); + } + + string configurationFile = this.GetConfigurationFileName(); + string configurationPath = this.Combine(this.PackageDirectory, "config", configurationFile); + string configText = await this.fileSystem.File.ReadAllTextAsync(configurationPath); + const string rateSection = "intrate,fprate:"; + if (!configText.Contains(rateSection, StringComparison.OrdinalIgnoreCase)) + { + throw new WorkloadException($"SPEC CPU configuration '{configurationPath}' is missing the '{rateSection}' section required for processor affinity."); + } + + string bind = useCopyNumber ? string.Empty : $" bind = {string.Join(",", cores)}\n"; + string submit = $" submit = numactl --localalloc --physcpubind={copyCpu} -- $command"; + configText = configText.Replace(rateSection, $"{rateSection}\n{bind}{submit}", StringComparison.OrdinalIgnoreCase); + + string configId = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join(",", cores)))).Substring(0, 16); + string pinnedConfigFile = $"{Path.GetFileNameWithoutExtension(configurationFile)}-affinity-{configId}.cfg"; + await this.fileSystem.File.WriteAllTextAsync(this.Combine(this.PackageDirectory, "config", pinnedConfigFile), configText, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "ConfigGenerated", ("configurationFile", pinnedConfigFile), ("cpuIds", string.Join(",", cores)), ("submit", submit)); + return pinnedConfigFile; + } + + private static bool TryParseNumaTopology(string topology, IReadOnlyList selectedCpus, out IReadOnlyList parsedNodes) + { + parsedNodes = null; + Match header = Regex.Match(topology, @"(?m)^available:[ \t]*(?\d+)[ \t]+nodes\b"); + if (!header.Success || !int.TryParse(header.Groups["count"].Value, out int nodeCount) || nodeCount == 0) + { + return false; + } + + Dictionary nodes = new Dictionary(); + MatchCollection entries = Regex.Matches(topology, @"(?m)^node[ \t]+(?\d+)[ \t]+(?cpus|size|free):[ \t]*(?[^\r\n]*)"); + foreach (Match entry in entries) + { + if (!int.TryParse(entry.Groups["id"].Value, out int nodeId)) + { + return false; + } + + if (!nodes.TryGetValue(nodeId, out NumaNode node)) + { + node = new NumaNode(nodeId); + nodes.Add(nodeId, node); + } + + string field = entry.Groups["field"].Value; + string value = entry.Groups["value"].Value.Trim(); + if (field == "cpus") + { + if (node.Cpus != null) + { + return false; + } + + List cpus = new List(); + foreach (string cpu in value.Split((char[])null, StringSplitOptions.RemoveEmptyEntries)) + { + if (!int.TryParse(cpu, out int cpuId) || cpuId < 0) + { + return false; + } + + cpus.Add(cpuId); + } + + node.Cpus = cpus; + } + else + { + Match memory = Regex.Match(value, @"^(?\d+(?:\.\d+)?)[ \t]+MB$"); + if (!memory.Success || !double.TryParse(memory.Groups["mb"].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out double megabytes)) + { + return false; + } + + if (field == "size") + { + if (node.SizeMb.HasValue) + { + return false; + } + + node.SizeMb = megabytes; + } + else + { + if (node.FreeMb.HasValue) + { + return false; + } + + node.FreeMb = megabytes; + } + } + } + + if (nodes.Count != nodeCount || nodes.Values.Any(node => node.Cpus == null || !node.SizeMb.HasValue || !node.FreeMb.HasValue)) + { + return false; + } + + if (nodes.Values.SelectMany(node => node.Cpus).Distinct().Count() != nodes.Values.Sum(node => node.Cpus.Count) + || selectedCpus.Any(cpu => nodes.Values.Count(node => node.Cpus.Contains(cpu)) != 1)) + { + return false; + } + + parsedNodes = nodes.Values.OrderBy(node => node.Id).ToList(); + return true; + } + + private void LogSpecCpuStep(EventContext telemetryContext, string step, params (string Name, object Value)[] details) + { + EventContext stepContext = telemetryContext.Clone() + .AddContext("scenario", this.Scenario ?? string.Empty) + .AddContext("specProfile", this.SpecProfile ?? string.Empty) + .AddContext("benchmarks", this.Benchmarks ?? string.Empty) + .AddContext("copies", this.CopyCount) + .AddContext("pinningEnabled", !string.IsNullOrWhiteSpace(this.NumaProcessorAffinity)); + + foreach ((string name, object value) in details) + { + stepContext.AddContext(name, value); + } + + this.Logger.LogMessage($"{this.TypeName}.{step}", LogLevel.Information, stepContext); + } + + private void LogSpecCpuFailure(EventContext telemetryContext, string stage, Exception exception) + { + EventContext errorContext = telemetryContext.Clone() + .AddContext("scenario", this.Scenario ?? string.Empty) + .AddContext("specProfile", this.SpecProfile ?? string.Empty) + .AddContext("stage", stage) + .AddError(exception); + + this.Logger.LogMessage($"{this.TypeName}.Failed", LogLevel.Error, errorContext); + } + private async Task WriteSpecCpuConfigAsync(CancellationToken cancellationToken) { string configurationFile = this.GetConfigurationFileName(); @@ -710,6 +1041,38 @@ public bool SpecCpuInitialized } } + private sealed class NumaNode + { + public NumaNode(int id) + { + this.Id = id; + } + + public int Id { get; } + + public List Cpus { get; set; } + + public double? SizeMb { get; set; } + + public double? FreeMb { get; set; } + } + + private sealed class NumaTopologySnapshot + { + public NumaTopologySnapshot(IReadOnlyList nodes, IReadOnlyList selectedCpus, DateTime capturedAt) + { + this.Nodes = nodes; + this.SelectedCpus = selectedCpus; + this.CapturedAt = capturedAt; + } + + public IReadOnlyList Nodes { get; } + + public IReadOnlyList SelectedCpus { get; } + + public DateTime CapturedAt { get; } + } + private static class SpecCpuConfigPlaceholder { public const string BaseOptimizingFlags = "$BaseOptimizingFlags$"; diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs index 63ab50d5c8..c85cdd7da7 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs @@ -58,6 +58,33 @@ public void MetricsCsvFileLoggerUsesTheExpectedColumnNamesForTheCsvFiles() CollectionAssert.AreEqual(expectedColumnHeaders, MetricsCsvFileLogger.CsvFields.Select(field => field.ColumnName)); } + [Test] + public void MetricsCsvFileLoggerDoesNotExportNumaCpuListsFromMetricMetadata() + { + EventContext context = new EventContext(Guid.NewGuid(), new Dictionary + { + ["toolName"] = "SPECcpu", + ["scenarioName"] = "ExecuteSPECBenchmark", + ["metricName"] = "node-cpus-0", + ["metricValue"] = 4, + ["metricUnit"] = "count", + ["metadata_metrics"] = new Dictionary + { + ["cpuIds"] = "0,1,48,49", + ["selectedCpuIds"] = "0,48", + ["processorAffinity"] = "0,1,48,49" + } + }); + + string csv = MetricsCsvFileLogger.CreateMessage(context); + + StringAssert.Contains("\"SPECcpu\"", csv); + StringAssert.Contains("\"node-cpus-0\",\"4\",\"count\"", csv); + StringAssert.DoesNotContain("0,1,48,49", csv); + StringAssert.DoesNotContain("0,48", csv); + StringAssert.DoesNotContain("cpuIds", csv); + } + [Test] public void MetricsCsvFileLoggerWritesTheExpectedCsvToFiles_1() { diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs index 2d35af8ede..9510e9179d 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs @@ -57,6 +57,24 @@ public void CreateProcessWithAffinityCreatesExpectedBashCommandOnLinux() } } + [Test] + public void CreateProcessAffinityOverloadCreatesExpectedBashCommandOnLinux() + { + this.SetupDefaults(PlatformID.Unix); + + ProcessAffinityConfiguration config = ProcessAffinityConfiguration.Create(PlatformID.Unix, new[] { 0, 1, 2 }); + + using (IProcessProxy process = this.mockFixture.ProcessManager.CreateProcess( + "bash -c", + "myworkload --option1=value", + "/home/user/workdir", + config)) + { + Assert.AreEqual("bash -c \"numactl -C 0-2 myworkload --option1=value\"", process.StartInfo.FileName); + Assert.AreEqual("/home/user/workdir", process.StartInfo.WorkingDirectory); + } + } + [Test] public void CreateProcessWithAffinityThrowsOnNullConfiguration() { @@ -109,6 +127,26 @@ public void CreateElevatedProcessWithAffinityCreatesExpectedCommandOnLinux() } } + [Test] + public void CreateProcessElevatedAffinityOverloadCreatesExpectedCommandOnLinux() + { + this.SetupDefaults(PlatformID.Unix); + + ProcessAffinityConfiguration config = ProcessAffinityConfiguration.Create(PlatformID.Unix, new[] { 0, 1, 2 }); + + using (IProcessProxy process = this.mockFixture.ProcessManager.CreateProcess( + PlatformID.Unix, + "bash -c", + "myworkload --option1=value", + "/home/user/workdir", + config)) + { + Assert.AreEqual("sudo", process.StartInfo.FileName); + Assert.AreEqual("bash -c \"numactl -C 0-2 myworkload --option1=value\"", process.StartInfo.Arguments); + Assert.AreEqual("/home/user/workdir", process.StartInfo.WorkingDirectory); + } + } + [Test] public void CreateElevatedProcessWithAffinityHandlesComplexArguments() { diff --git a/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs b/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs index 01344e3da2..15df14d3f0 100644 --- a/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs +++ b/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs @@ -154,6 +154,36 @@ public static IProcessProxy CreateElevatedProcessWithAffinity(this ProcessManage } } + /// + /// Creates a Linux process with CPU affinity binding to specific cores. + /// + /// The process manager used to create the process. + /// The command to run. + /// The command line arguments to supply to the command. + /// The working directory for the command. + /// The CPU affinity configuration specifying which cores to bind to. + /// A process proxy with CPU affinity applied via numactl. + public static IProcessProxy CreateProcess(this ProcessManager processManager, string command, string arguments, string workingDir, ProcessAffinityConfiguration affinityConfig) + { + return processManager.CreateProcessWithAffinity(command, arguments, workingDir, affinityConfig); + } + + /// + /// Creates a Linux process with CPU affinity binding and elevated privileges when needed. + /// + /// The process manager used to create the process. + /// The OS platform. + /// The command to run. + /// The command line arguments to supply to the command. + /// The working directory for the command. + /// The CPU affinity configuration specifying which cores to bind to. + /// The username to use for running the command (Linux only). + /// A process proxy with CPU affinity and elevated privileges applied. + public static IProcessProxy CreateProcess(this ProcessManager processManager, PlatformID platform, string command, string arguments, string workingDir, ProcessAffinityConfiguration affinityConfig, string username = null) + { + return processManager.CreateElevatedProcessWithAffinity(platform, command, arguments, workingDir, affinityConfig, username); + } + /// /// Applies Windows CPU affinity to a running process. /// This should be called after the process has started. diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json index 92efd89f00..b5b9e95613 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json @@ -10,7 +10,7 @@ "RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=05:00:00,(64-cores)=10:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "AzureLinux,CentOS,Debian,OpenSuse,RedHat,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24. SPEC CPU 2017 was not initially designed for win-arm64.", + "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24. SPEC CPU 2017 was not initially designed for win-arm64. Optional NumaProcessorAffinity is Linux-only: provide one distinct CPU ID per copy (e.g. 0-3 for CopyCount=4), or leave empty to disable pinning.", "Limitations": "The following benchmarks don't work on Windows: 507, 511, 521, 526, 527, 538" }, "Parameters": { @@ -20,6 +20,7 @@ "RunPeak": false, "ThreadCount": "{LogicalCoreCount}", "CopyCount": "{LogicalCoreCount}", + "NumaProcessorAffinity": "", "BaseOptimizingFlags": "-O2 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches", "PeakOptimizingFlags": "-O3 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches" }, @@ -35,6 +36,7 @@ "RunPeak": "$.Parameters.RunPeak", "ThreadCount": "$.Parameters.ThreadCount", "CopyCount": "$.Parameters.CopyCount", + "NumaProcessorAffinity": "$.Parameters.NumaProcessorAffinity", "BaseOptimizingFlags": "$.Parameters.BaseOptimizingFlags", "PeakOptimizingFlags": "$.Parameters.PeakOptimizingFlags" } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json index 5a0305b8ed..281f8e24d0 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json @@ -10,7 +10,7 @@ "RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=08:00:00,(64-cores)=10:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", "SupportedOperatingSystems": "AzureLinux,CentOS,Debian,OpenSuse,RedHat,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24. SPEC CPU 2017 was not initially designed for win-arm64.", + "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24. SPEC CPU 2017 was not initially designed for win-arm64. Optional NumaProcessorAffinity is Linux-only: provide one distinct CPU ID per copy (e.g. 0-3 for CopyCount=4), or leave empty to disable pinning.", "Limitations": "The following benchmarks don't work on Windows: 500, 502, 520, 523, 531" }, "Parameters": { @@ -20,6 +20,7 @@ "RunPeak": false, "ThreadCount": "{LogicalCoreCount}", "CopyCount": "{LogicalCoreCount}", + "NumaProcessorAffinity": "", "BaseOptimizingFlags": "-O2 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches", "PeakOptimizingFlags": "-O3 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches" }, @@ -35,6 +36,7 @@ "RunPeak": "$.Parameters.RunPeak", "ThreadCount": "$.Parameters.ThreadCount", "CopyCount": "$.Parameters.CopyCount", + "NumaProcessorAffinity": "$.Parameters.NumaProcessorAffinity", "BaseOptimizingFlags": "$.Parameters.BaseOptimizingFlags", "PeakOptimizingFlags": "$.Parameters.PeakOptimizingFlags" } diff --git a/website/docs/workloads/speccpu/speccpu-profiles.md b/website/docs/workloads/speccpu/speccpu-profiles.md index c4e945e651..9b285be645 100644 --- a/website/docs/workloads/speccpu/speccpu-profiles.md +++ b/website/docs/workloads/speccpu/speccpu-profiles.md @@ -44,6 +44,7 @@ for evaluating the performance of the CPU for processing calculations. | RunPeak | Optional. True to run the workload 'Peak' scenario, False to run the workload 'Base' scenario. | false (Base) | | ThreadCount | Optional. Determines the number of threads to use for running the benchmark. | # logical processors | | CopyCount | Optional. Determines the number of copies of the benchmark to run concurrently. | # logical processors | + | NumaProcessorAffinity | Optional. Linux SPECrate only (intrate/fprate). Assign one distinct CPU ID to each copy; requires `numactl`. Leave empty to run without pinning. | empty (no pinning) | | BaseOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Base' scenario |
  • X64 systems: -g -O2 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O2 -march=armv8.2-a -flto -frecord-gcc-switches
| | PeakOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Peak' scenario |
  • X64 systems: -g -O3 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O3 -march=armv8.2-a -flto -frecord-gcc-switches
| @@ -64,6 +65,7 @@ for evaluating the performance of the CPU for processing calculations. * **Notes** On Windows, only the following benchmarks are supported: 503.bwaves_r, 508.namd_r, 510.parest_r, 519.lbm_r, 544.nab_r, 549.fotonik3d_r, 554.roms_r. + The Linux `NumaProcessorAffinity` examples in the INTRATE section below also apply to this profile when `PERF-SPECCPU-FPRATE.json` is used instead. ``` bash # Execute the workload profile @@ -155,6 +157,7 @@ for evaluating the performance of the CPU for processing calculations. | RunPeak | Optional. True to run the workload 'Peak' scenario, False to run the workload 'Base' scenario. | false (Base) | | ThreadCount | Optional. Determines the number of threads to use for running the benchmark. | # logical processors | | CopyCount | Optional. Determines the number of copies of the benchmark to run concurrently. | # logical processors | + | NumaProcessorAffinity | Optional. Linux SPECrate only (intrate/fprate). Assign one distinct CPU ID to each copy; requires `numactl`. Leave empty to run without pinning. | empty (no pinning) | | BaseOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Base' scenario |
  • X64 systems: -g -O2 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O2 -march=armv8.2-a -flto -frecord-gcc-switches
| | PeakOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Peak' scenario |
  • X64 systems: -g -O3 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O3 -march=armv8.2-a -flto -frecord-gcc-switches
| @@ -174,10 +177,17 @@ for evaluating the performance of the CPU for processing calculations. * **Notes** On Windows, only the following benchmarks are supported: 505.mcf_r, 525.x264_r, 541.leela_r, 548.exchange2_r, 557.xz_r. + On Linux, check the CPU-to-node mapping with `numactl --hardware` before choosing CPU IDs. `NumaProcessorAffinity` must contain exactly as many distinct CPU IDs as `CopyCount`. Pinned copies use `numactl --localalloc` for local memory allocation. The CPU IDs below are examples; use IDs appropriate for your machine. ``` bash - # Execute the workload profile + # Empty NumaProcessorAffinity (default): run without pinning, as before. ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --iterations=1 --logger=csv + + # Single NUMA node: CPUs 0-3 must all belong to the same node. + ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --iterations=1 --parameters="CopyCount=4,,,NumaProcessorAffinity=0-3" --logger=csv + + # Two NUMA nodes: CPUs 0-1 on node 0 and 48-49 on node 1. + ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --iterations=1 --parameters="CopyCount=4,,,NumaProcessorAffinity=0,1,48,49" --logger=csv ``` ## PERF-SPECCPU-INTSPEED.json