From f9f2fe5de880ddb27ca76cc894479bf98d126970 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Fri, 9 Oct 2026 15:42:14 -0700 Subject: [PATCH 1/7] Migrate IPMIUtil and Windows ETW monitors to open source Move both monitor implementations, tests, profiles, and documentation into the public repository. Rename ETWMonitor to WindowsETWMonitor and advance the package version for the coordinated internal cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- VERSION | 2 +- .../WindowsETWMonitorProfileTests.cs | 42 + .../profiles/MONITORS-AZURE-HOST.json | 51 + .../profiles/MONITORS-ETW-KERNEL.json | 32 + .../profiles/MONITORS-ETW-NONKERNEL.json | 30 + .../profiles/MONITORS-ETW.json | 47 + .../profiles/MONITORS-IPMIUTIL.json | 76 + .../ETW/WindowsETWMonitorTests.cs | 265 ++ .../ipmiutil/decodedIpmiUtilSelOutput.txt | 554 ++++ .../ipmiutil_sel_decoded_example_1.txt | 239 ++ .../ipmiutil_sel_decoded_example_2.txt | 1946 +++++++++++++ .../ipmiutil/ipmiutil_sel_example_1.txt | 240 ++ .../ipmiutil/ipmiutil_sensor_example_1.txt | 112 + .../ipmiutil/ipmiutil_sensor_example_2.txt | 163 ++ .../ipmiutil/ipmiutil_sensor_example_3.txt | 112 + .../Ipmiutil/IpmiUtilSelMetricsParserTests.cs | 408 +++ .../Ipmiutil/IpmiUtilSelMonitorTests.cs | 2440 +++++++++++++++++ .../IpmiUtilSensorMetricsParserTests.cs | 374 +++ .../Ipmiutil/IpmiUtilSensorMonitorTests.cs | 285 ++ .../VirtualClient.Monitors/AssemblyInfo.cs | 2 + .../ETW/WindowsETWMonitor.cs | 748 +++++ .../Ipmiutil/IpmiUtilSelMetricsParser.cs | 453 +++ .../Ipmiutil/IpmiUtilSelMonitor.cs | 1463 ++++++++++ .../Ipmiutil/IpmiUtilSensorMetricsParser.cs | 164 ++ .../Ipmiutil/IpmiUtilSensorMonitor.cs | 241 ++ website/docs/monitors/ipmiutil.md | 120 + website/docs/monitors/monitor-profiles.md | 81 + .../docs/monitors/systemevents/windows-etw.md | 124 + website/docs/overview/overview.md | 1 + 29 files changed, 10814 insertions(+), 1 deletion(-) create mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/ETW/WindowsETWMonitorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/decodedIpmiUtilSelOutput.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_1.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_2.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_example_1.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_1.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_2.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_3.txt create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMetricsParserTests.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMetricsParserTests.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMonitorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors/ETW/WindowsETWMonitor.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMetricsParser.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMonitor.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMetricsParser.cs create mode 100644 src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMonitor.cs create mode 100644 website/docs/monitors/ipmiutil.md create mode 100644 website/docs/monitors/systemevents/windows-etw.md diff --git a/VERSION b/VERSION index 3cf57515c3..fa8da2074b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.5.7 +3.5.8 diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs new file mode 100644 index 0000000000..d9e6d2521b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Linq; + using NUnit.Framework; + using VirtualClient.Contracts; + using VirtualClient.Monitors; + using VirtualClient.TestExtensions; + + [TestFixture] + [Category("Functional")] + public class WindowsETWMonitorProfileTests + { + private DependencyFixture mockFixture; + + [OneTimeSetUp] + public void SetupFixture() + { + this.mockFixture = new DependencyFixture(); + ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); + } + + [Test] + [TestCase("MONITORS-ETW.json", 2)] + [TestCase("MONITORS-ETW-KERNEL.json", 1)] + [TestCase("MONITORS-ETW-NONKERNEL.json", 1)] + public void WindowsETWMonitorProfileParametersAreInlinedCorrectly(string profile, int expectedMonitorCount) + { + this.mockFixture.Setup(PlatformID.Win32NT); + + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + Assert.AreEqual(expectedMonitorCount, executor.Profile.Monitors.Count); + Assert.IsTrue(executor.Profile.Monitors.All(monitor => monitor.Type == nameof(WindowsETWMonitor))); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json new file mode 100644 index 0000000000..c9dae75011 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json @@ -0,0 +1,51 @@ +{ + "Description": "Default Azure Host Monitors", + "Metadata": { + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows", + "PreliminaryRequirements": "This profile requires direct access to an IPMI-capable Azure host or blade." + }, + "Parameters": {}, + "Monitors": [ + { + "Type": "IpmiUtilSensorMonitor", + "Parameters": { + "Scenario": "CaptureSensorData", + "MonitorFrequency": "00:00:30" + } + }, + { + "Type": "IpmiUtilSelMonitor", + "Parameters": { + "Scenario": "CaptureSelData", + "ClearSel": false, + "EventsFilter": "Correctable ECC,FRB2,FRB3,IERR,Predictive failure,Uncorrectable ECC,Bus Error,Fatal NMI", + "MonitorFrequency": "00:01:00" + } + }, + { + "Type": "WindowsPerformanceCounterMonitor", + "Parameters": { + "Scenario": "CaptureCounters", + "MonitorFrequency": "00:05:00", + "MonitorWarmupPeriod": "00:05:00", + "Counters01": "Hyper-V Hypervisor Logical Processor=\\(_Total\\)", + "Counters02": "Hyper-V Hypervisor Root Virtual Processor=\\(_Total\\)\\\\% (Guest|Hypervisor|Total) Run Time", + "Counters03": "Hyper-V Hypervisor Root Virtual Processor=\\(_Total\\)\\\\(CPU Wait Time Per Dispatch|Hardware Interrupts/sec|Hypercalls Cost|Hypercalls/sec|Long Spin Wait Hypercalls/sec)", + "Counters04": "Hyper-V Hypervisor Root Virtual Processor=\\(_Total\\)\\\\(Total Intercepts Cost|Total Intercepts/sec)", + "Counters05": "Hyper-V Hypervisor Virtual Processor=\\(_Total\\)\\\\% (Guest|Hypervisor|Total|VTL2) Run Time", + "Counters06": "Hyper-V Hypervisor Virtual Processor=\\(_Total\\)\\\\(CPU Wait Time Per Dispatch|Hardware Interrupts/sec|Hypercalls Cost|Hypercalls/sec|Logical Processor Migrations/sec|Long Spin Wait Hypercalls/sec)", + "Counters07": "Hyper-V Hypervisor Virtual Processor=\\(_Total\\)\\\\(Nested Page Fault Intercepts/sec|Posted Interrupt Notifications/sec|Posted Interrupt Scans/sec|Total Intercepts Cost|Total Intercepts/sec)", + "Counters08": "IPv4=.", + "Counters09": "Memory=(Available|Cache|Committed) Bytes", + "Counters10": "Memory=Faults/sec", + "Counters11": "Memory=(Page Reads/sec|Page Writes/sec|Pages/sec|Pages Input/sec|Pages Output/sec)", + "Counters12": "PhysicalDisk=\\(_Total\\)", + "Counters13": "Processor Information=\\(_Total\\)", + "Counters14": "Processor Information=\\([0-9,]+\\)\\\\% (C[0-9]+|Idle|Interrupt|Privileged|Processor|User) Time", + "Counters15": "System=.", + "Comments": "The counters defined are formatted as {category}={counter_match_expression}. The match expression is a regular expression to allow for handling variability of counters available on different systems. JSON content has eccentricities. Use 4-backslashes to represent a single backslash and 2-backslashes to represent a regular expression character escape." + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json new file mode 100644 index 0000000000..4f90aa962d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json @@ -0,0 +1,32 @@ +{ + "Description": "Capturing the ETW traces for Kernel provider", + "Metadata": { + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows" + }, + "Parameters": { + "ProfilingEnabled": true, + "ProfilingMode": "Interval", + "ProfilingPeriod": "00:05:00", + "ProfilingInterval": "00:00:30", + "ProfilingWarmUpPeriod": "00:00:10" + }, + "Monitors": [ + { + "Type": "WindowsETWMonitor", + "Parameters": { + "Scenario": "CaptureKernelETWTraces", + "ProfilingEnabled": "$.Parameters.ProfilingEnabled", + "ProfilingMode": "$.Parameters.ProfilingMode", + "ProfilingPeriod": "$.Parameters.ProfilingPeriod", + "ProfilingInterval": "$.Parameters.ProfilingInterval", + "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", + "ProviderType": "Kernel", + "Provider": "Windows Kernel", + "ProviderKeywords": "ImageLoad,Process", + "ProviderEvents": "ImageLoad,ProcessStart,ProcessStop", + "JsonFilters": "" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json new file mode 100644 index 0000000000..003f7b4340 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json @@ -0,0 +1,30 @@ +{ + "Description": "Capturing the ETW traces for Nonkernel provider", + "Metadata": { + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows" + }, + "Parameters": { + "ProfilingEnabled": true, + "ProfilingMode": "Interval", + "ProfilingPeriod": "00:05:00", + "ProfilingInterval": "00:00:30", + "ProfilingWarmUpPeriod": "00:00:10" + }, + "Monitors": [ + { + "Type": "WindowsETWMonitor", + "Parameters": { + "Scenario": "CaptureNonKernelETWTraces", + "ProfilingEnabled": "$.Parameters.ProfilingEnabled", + "ProfilingMode": "$.Parameters.ProfilingMode", + "ProfilingPeriod": "$.Parameters.ProfilingPeriod", + "ProfilingInterval": "$.Parameters.ProfilingInterval", + "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", + "ProviderType": "NonKernel", + "Provider": "Microsoft-Windows-DotNETRuntime", + "JsonFilters": "{'EventName':['GCSetGCHandle','GCTriggered'],'Level':['4','2']}" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json new file mode 100644 index 0000000000..6df147d78f --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json @@ -0,0 +1,47 @@ +{ + "Description": "Capturing the ETW traces for Kernel and NonKernel events", + "Metadata": { + "SupportedPlatforms": "win-x64,win-arm64", + "SupportedOperatingSystems": "Windows" + }, + "Parameters": { + "ProfilingEnabled": true, + "ProfilingMode": "Interval", + "ProfilingPeriod": "00:05:00", + "ProfilingInterval": "00:00:30", + "ProfilingWarmUpPeriod": "00:00:10" + }, + "Monitors": [ + { + "Type": "WindowsETWMonitor", + "Parameters": { + "Scenario": "CaptureNonKernelETWTraces", + "ProfilingEnabled": "$.Parameters.ProfilingEnabled", + "ProfilingMode": "$.Parameters.ProfilingMode", + "ProfilingPeriod": "$.Parameters.ProfilingPeriod", + "ProfilingInterval": "$.Parameters.ProfilingInterval", + "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", + "ProviderType": "NonKernel", + "Provider": "Microsoft-Windows-DotNETRuntime", + "JsonFilters": "{'EventName':['GCSetGCHandle','GCTriggered'],'Level':['4','2']}" + + } + }, + { + "Type": "WindowsETWMonitor", + "Parameters": { + "Scenario": "CaptureKernelETWTraces", + "ProfilingEnabled": "$.Parameters.ProfilingEnabled", + "ProfilingMode": "$.Parameters.ProfilingMode", + "ProfilingPeriod": "$.Parameters.ProfilingPeriod", + "ProfilingInterval": "$.Parameters.ProfilingInterval", + "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", + "ProviderType": "Kernel", + "Provider": "Windows Kernel", + "ProviderKeywords": "ImageLoad,Process", + "ProviderEvents": "ImageLoad,ProcessStart,ProcessStop", + "JsonFilters": "" + } + } + ] +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json new file mode 100644 index 0000000000..bd4fc9ed57 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json @@ -0,0 +1,76 @@ +{ + "Description": "Captures BMC sensor readings and System Event Log (SEL) events using ipmiutil.", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:05:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "AzureLinux,CentOS,Debian,OpenSuse,RedHat,Ubuntu,Windows", + "PreliminaryRequirements": "The target system must expose a supported IPMI/BMC interface. Linux package installation requires access to the configured operating-system package repositories. Windows package installation requires access to the Virtual Client packages store." + }, + "Parameters": { + "MonitorFrequency": "00:01:00", + "MonitorWarmupPeriod": "00:00:00", + "MonitorIterations": -1, + "ClearSel": false, + "EventsFilter": "MAJ|CRT", + "CaptureSuspectedEvents": false, + "IncludeWarningsInSuspectedEvents": true, + "EnableSelDecoding": false, + "DecoderType": "", + "DecoderArguments": "", + "DecoderPackageName": "", + "DecoderExecutableName": "" + }, + "Monitors": [ + { + "Type": "IpmiUtilSensorMonitor", + "Parameters": { + "Scenario": "CaptureBmcSensorData", + "PackageName": "host.monitors", + "MonitorFrequency": "$.Parameters.MonitorFrequency", + "MonitorWarmupPeriod": "$.Parameters.MonitorWarmupPeriod", + "MonitorIterations": "$.Parameters.MonitorIterations" + } + }, + { + "Type": "IpmiUtilSelMonitor", + "Parameters": { + "Scenario": "CaptureBmcSelData", + "PackageName": "host.monitors", + "MonitorFrequency": "$.Parameters.MonitorFrequency", + "MonitorWarmupPeriod": "$.Parameters.MonitorWarmupPeriod", + "MonitorIterations": "$.Parameters.MonitorIterations", + "ClearSel": "$.Parameters.ClearSel", + "EventsFilter": "$.Parameters.EventsFilter", + "CaptureSuspectedEvents": "$.Parameters.CaptureSuspectedEvents", + "IncludeWarningsInSuspectedEvents": "$.Parameters.IncludeWarningsInSuspectedEvents", + "EnableSelDecoding": "$.Parameters.EnableSelDecoding", + "DecoderType": "$.Parameters.DecoderType", + "DecoderArguments": "$.Parameters.DecoderArguments", + "DecoderPackageName": "$.Parameters.DecoderPackageName", + "DecoderExecutableName": "$.Parameters.DecoderExecutableName" + } + } + ], + "Dependencies": [ + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallIpmiutilWindowsPackage", + "BlobContainer": "packages", + "BlobName": "host.monitors.1.0.1.zip", + "PackageName": "host.monitors", + "Extract": true, + "SupportedPlatforms": "win-x64,win-arm64" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallIpmiutilLinuxPackage", + "Packages": "ipmiutil", + "AllowUpgrades": true, + "SupportedPlatforms": "linux-x64,linux-arm64" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/ETW/WindowsETWMonitorTests.cs b/src/VirtualClient/VirtualClient.Monitors.UnitTests/ETW/WindowsETWMonitorTests.cs new file mode 100644 index 0000000000..1c044a73ff --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/ETW/WindowsETWMonitorTests.cs @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Diagnostics.Tracing.Session; + using NUnit.Framework; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + + [TestFixture] + [Category("Unit")] + [Platform(Exclude = "Unix,Linux,MacOsX")] + public class WindowsETWMonitorTests : MockFixture + { + private MockFixture mockFixture; + + [SetUp] + public void SetupTest() + { + this.mockFixture = this; + this.mockFixture.Setup(PlatformID.Win32NT); + + this.mockFixture.Parameters = new Dictionary() + { + { "ProfilingEnabled", true} + }; + + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + InMemoryProcess process = new InMemoryProcess() + { + StartInfo = new System.Diagnostics.ProcessStartInfo() + { + FileName = command, + Arguments = arguments, + WorkingDirectory = workingDir + }, + OnHasExited = () => true, + ExitCode = 0, + OnStart = () => true, + StandardOutput = new ConcurrentBuffer() + }; + + return process; + }; + } + + [Test] + public void WindowsETWMonitorThrowsWhenRunningWithoutAdminPrivileges() + { + if (TraceEventSession.IsElevated() == false) + { + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + this.mockFixture.Parameters["JsonFilters"] = "testing"; + MonitorException exception = Assert.Throws(() => monitor.InitializeAsync(EventContext.None, CancellationToken.None)); + Assert.AreEqual(ErrorReason.Unauthorized, exception.Reason); + } + } + } + + [Test] + public void WindowsETWMonitorThrowsWhenJsonFiltersParameterIsInImproperFormat() + { + this.mockFixture.Parameters["JsonFilters"] = "testing"; + if (TraceEventSession.IsElevated() == true) + { + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + MonitorException exception = Assert.Throws(() => monitor.InitializeAsync(EventContext.None, CancellationToken.None)); + Assert.AreEqual(ErrorReason.InvalidProfileDefinition, exception.Reason); + } + } + } + + [Test] + public void WindowsETWMonitorThrowsOnValidateWhenProviderTypeParameterIsEmpty() + { + this.mockFixture.Parameters["ProviderType"] = string.Empty; + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + MonitorException exception = Assert.Throws(() => monitor.Validate()); + Assert.AreEqual(ErrorReason.InvalidProfileDefinition, exception.Reason); + } + } + + [Test] + public void WindowsETWMonitorThrowsOnValidateWhenProviderParameterIsEmpty() + { + this.mockFixture.Parameters["ProviderType"] = "NonKernel"; + this.mockFixture.Parameters["Provider"] = string.Empty; + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + MonitorException exception = Assert.Throws(() => monitor.Validate()); + Assert.AreEqual(ErrorReason.InvalidProfileDefinition, exception.Reason); + } + } + + [Test] + public void WindowsETWMonitorThrowsOnValidateWhenProviderTypeIsKernelAndKernelProviderKeywordsAreEmpty() + { + this.mockFixture.Parameters["ProviderType"] = "Kernel"; + this.mockFixture.Parameters["Provider"] = "Windows Kernel"; + this.mockFixture.Parameters["ProviderKeywords"] = string.Empty; + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + MonitorException exception = Assert.Throws(() => monitor.Validate()); + Assert.AreEqual(ErrorReason.InvalidProfileDefinition, exception.Reason); + } + } + + + [Test] + [Ignore("Invalid setup. This test checks to see if the application is running with elevated privileges. This should not be required for a unit test.")] + public async Task WindowsETWMonitorExecuteProfilingOnDemand() + { + int count = 0; + this.mockFixture.Parameters["ProfilingMode"] = "OnDemand"; + using (CancellationTokenSource cancellationSource = new CancellationTokenSource()) + { + Task timeoutTask = Task.Run(async () => + { + DateTime finishTime = DateTime.UtcNow.AddSeconds(20); + while (DateTime.UtcNow < finishTime) + { + await Task.Delay(100); + } + + cancellationSource.Cancel(); + }); + + Task executorTask = Task.Run(async () => + { + await Task.Delay(2000).ConfigureAwait(false); + using (TestExecutor component = new TestExecutor(this.mockFixture)) + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(component, cancellationSource.Token)) + { + while (!cancellationSource.IsCancellationRequested) + { + await Task.Delay(500); + } + } + } + }); + + Task etwMonitorTask = Task.Run(async () => + { + this.mockFixture.Parameters["ProviderType"] = "NonKernel"; + this.mockFixture.Parameters["Provider"] = "Microsoft-Windows-DotNETRuntime"; + this.mockFixture.Parameters["ProfilingWarmUpPeriod"] = "00:00:00"; + this.mockFixture.Parameters["JsonFilters"] = ""; + + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + await monitor.ExecuteAsync(cancellationSource.Token); + count = monitor.etwData != null ? monitor.etwData.Count : 0; + } + }); + + Task createEventTask = Task.Run(async () => + { + using (IProcessProxy process = this.mockFixture.ProcessManager.CreateElevatedProcess(this.mockFixture.Platform, "cmd.exe")) + { + await Task.Delay(10000); + await process.StartAndWaitAsync(cancellationSource.Token, TimeSpan.FromSeconds(30)) + .ConfigureAwait(false); + } + }); + + await Task.WhenAll(timeoutTask, executorTask, etwMonitorTask, createEventTask).ConfigureAwait(false); + + // await Task.WhenAll(etwMonitorTask, etwMonitorTask2, executorTask, timeoutTask); + + } + + Assert.True(count > 0); + } + + [Test] + [Ignore("Invalid setup. This test checks to see if the application is running with elevated privileges. This should not be required for a unit test.")] + public async Task WindowsETWMonitorExecuteProfilingInterval() + { + int count = 0; + this.mockFixture.Parameters["ProfilingMode"] = "Interval"; + this.mockFixture.Parameters["ProfilingPeriod"] = "00:00:20"; + this.mockFixture.Parameters["ProfilingInterval"] = "00:00:00"; + using (CancellationTokenSource cancellationSource = new CancellationTokenSource()) + { + Task timeoutTask = Task.Run(async () => + { + DateTime finishTime = DateTime.UtcNow.AddSeconds(20); + while (DateTime.UtcNow < finishTime) + { + await Task.Delay(100); + } + + cancellationSource.Cancel(); + }); + + Task etwMonitorTask = Task.Run(async () => + { + this.mockFixture.Parameters["ProviderType"] = "Kernel"; + this.mockFixture.Parameters["Provider"] = "Windows Kernel"; + this.mockFixture.Parameters["JsonFilters"] = ""; + this.mockFixture.Parameters["ProviderKeywords"] = "ImageLoad,Process"; + this.mockFixture.Parameters["ProviderEvents"] = "ImageLoad,ProcessStart,ProcessStop"; + using (TestWindowsETWMonitor monitor = new TestWindowsETWMonitor(this.mockFixture)) + { + await monitor.ExecuteAsync(cancellationSource.Token); + count = monitor.etwData != null ? monitor.etwData.Count : 0; + } + }); + + + Task createEventTask = Task.Run(async () => + { + using (IProcessProxy process = this.mockFixture.ProcessManager.CreateElevatedProcess(this.mockFixture.Platform, "cmd.exe")) + { + await Task.Delay(4000); + await process.StartAndWaitAsync(cancellationSource.Token, TimeSpan.FromSeconds(30)) + .ConfigureAwait(false); + } + }); + + await Task.WhenAll(timeoutTask, etwMonitorTask, createEventTask).ConfigureAwait(false); + + } + + Assert.True(count > 0); + } + + private class TestWindowsETWMonitor : WindowsETWMonitor + { + public TestWindowsETWMonitor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + } + + public new Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeAsync(telemetryContext, cancellationToken); + } + + public new void Validate() + { + base.Validate(); + } + + public new Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.ExecuteAsync(telemetryContext, cancellationToken); + } + + public Dictionary etwData => ETWData; + } + } +} diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/decodedIpmiUtilSelOutput.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/decodedIpmiUtilSelOutput.txt new file mode 100644 index 0000000000..3fc0d65b9c --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/decodedIpmiUtilSelOutput.txt @@ -0,0 +1,554 @@ + + 00000 + 734SHERLOCK77 + 7800000000000002 + 700000COBALT000001 + 10000IFWI000000000007 + 72000EF0000RV0000000003 + 760000TM000000000PC000003 + 30086426000000000JB000AD0004 ************************************************* + 5000000000CC00000000097 * ____ _ _____ _ _ * + 7600000DEADBEEF000000080057 * / ___|| |__ | ____|_ __| | ___ ___| | __ * + H 9000RBM000000000000003 7383 * \___ \| '_ \| _| | '__| | / _ \ / __| |/ / * + R 800000000000000OS00002 * ___) | | | | |___| | | |__| (_) | (__| < * + S 1360000000HRSD000000897 * |____/|_| |_|_____|_| |_____\___/ \___|_|\_\ * + D 715000BMC000000000000000027 ************************************************* + 366 4677500000000000BUGS0000007 + 0GSD207 200000000000000000000077 + 300007 712221774000000PSE00000817 + 3000000000JY0000008 + 7000SOC000000000005 + 400000000SEL00917 + 30000000008517 + 7000008477 + 708217 + +ID Timestamp (UTC) Sensor/OEM(Owner) Sensor Type Event Type Message Direction Record Type Raw Hex +---- ------------------- --------------------------------------------- -------------------------------- ------------------------ -------------------------------------------------- ---------- --------------- ------------------------------------------------ +0x1 2026-03-19 19:39:40 SEL(BMC)=0x03 Event Logging Disabled Sensor-Specific Discrete SEL Cleared ASSERTED System Event 01 00 02 fc 50 bc 69 20 00 04 10 03 6f 02 ff ff +0x2 2026-03-19 19:41:28 SoC_Status(BMC)=0x98 BMC OEM OEM Discrete POST Complete Timeout, Last Boot Status Code ASSERTED System Event 02 00 02 68 51 bc 69 20 00 04 df 98 70 aa e3 ff + Seen=0xE3 -> ERROR: + EFI_SW_DXE_BS_EC_BOOT_OPTION_LOAD_ERROR, Source: + AP-NS +0x3 2026-03-20 00:28:51 BMC_Health(BMC)=0xEF OEM Management Subsystem Health OEM Discrete I2C Recovery (Due to Bus Hang or Timeout): Bus ASSERTED System Event 03 00 02 c3 94 bc 69 20 00 04 e0 ef 70 aa 09 ff + ID=9, Error Code=255 +0x4 2026-03-20 17:16:42 SoC_Status(BMC)=0x98 BMC OEM OEM Discrete PLATFORM RESET REQUEST ASSERTED System Event 04 00 02 fa 80 bd 69 20 00 04 df 98 70 a6 ff ff +0x5 2026-03-20 17:16:42 Power_Status(BMC)=0x97 System ACPI Power State Sensor-Specific Discrete SOC Power State: S5/G2 "Soft Off" -> SLEEP_S5_N ASSERTED System Event 05 00 02 fa 80 bd 69 20 00 04 22 97 6f 05 ff ff + Asserted by SoC +0x6 2026-03-20 17:16:42 SoC_Status(BMC)=0x98 BMC OEM OEM Discrete SOC Sleep S5 ASSERTED System Event 06 00 02 fa 80 bd 69 20 00 04 df 98 70 a7 ff ff +0x7 2026-03-20 17:16:42 Generic Purpose(BMC)=0xD9 N/A N/A Unknown=8, [DECODE ERROR] N/A OEM Timestamped 07 00 d9 fa 80 bd 69 20 00 00 08 00 01 ff ff ff +0x8 2026-03-20 17:16:52 SoC_Status(BMC)=0x98 BMC OEM OEM Discrete SOC Sleep S5 DEASSERTED System Event 08 00 02 04 81 bd 69 20 00 04 df 98 f0 a7 ff ff +0x9 2026-03-20 17:16:53 Generic Purpose(BMC)=0xD9 N/A N/A Trust Mode Change -> Current Trust Mode: General N/A OEM Timestamped 09 00 d9 05 81 bd 69 20 00 00 04 01 01 01 02 00 + Purpose, New Trust Mode: General Purpose, Change + Method: Host OS Boot-up, Trust Mode Type: + Immediate, Trust JSON File: Available +0xA 2026-03-20 17:16:53 Power_Status(BMC)=0x97 System ACPI Power State Sensor-Specific Discrete SOC Power State: S0/G0 "Working" -> SOCPWRGOOD ASSERTED System Event 0a 00 02 05 81 bd 69 20 00 04 22 97 6f 00 ff ff + High +0xB 2026-03-20 17:16:54 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 0b 00 c0 06 81 bd 69 51 00 00 00 10 07 00 00 00 + HSP_SEL_TYPE_FIRMWARE_RESET_REASON -> + HSP_FIRMWARE_RESET_REASON_COLD_BOOT +0xC 2026-03-20 17:16:54 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 0c 00 c0 06 81 bd 69 51 00 00 00 01 c5 00 08 ff + HSP_SEL_TYPE_SECURITY_STATE -> mission_mode, + watchdog_enable, verify_svn, verify_signature, + profile=0x00, HspSecurityStateSecure, + HSPUnlockType=0xFF +0xD 2026-03-20 17:16:54 SoC_Status(BMC)=0x98 BMC OEM OEM Discrete PLATFORM RESET REQUEST DEASSERTED System Event 0d 00 02 06 81 bd 69 20 00 04 df 98 f0 a6 ff ff +0xE 2026-03-20 17:16:55 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 0e 00 c0 07 81 bd 69 51 00 00 00 11 01 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_SENDING_LOAD_SP1_COMMAND +0xF 2026-03-20 17:16:55 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 0f 00 c0 07 81 bd 69 51 00 00 00 11 02 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_FUSE_IMAGE_LOAD_RESPONSE_RECEIVED +0x10 2026-03-20 17:16:55 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 10 00 c0 07 81 bd 69 51 00 00 01 10 07 00 00 00 + HSP_SEL_TYPE_FIRMWARE_RESET_REASON -> + HSP_FIRMWARE_RESET_REASON_COLD_BOOT +0x11 2026-03-20 17:16:55 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 11 00 c0 07 81 bd 69 51 00 00 00 11 04 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_BOOT_STATUS_COMPLETE_RECEIVED_FROM_HSP_ + SECONDARY +0x12 2026-03-20 17:16:56 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 12 00 c0 08 81 bd 69 51 00 00 01 01 c5 00 08 ff + HSP_SEL_TYPE_SECURITY_STATE -> mission_mode, + watchdog_enable, verify_svn, verify_signature, + profile=0x00, HspSecurityStateSecure, + HSPUnlockType=0xFF +0x13 2026-03-20 17:16:56 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 13 00 c0 08 81 bd 69 51 00 00 00 04 00 00 00 00 + HSP_SEL_TYPE_SOC_SVN_READ_FROM_IMAGE -> SVN + Firmware Value=0x00 +0x14 2026-03-20 17:16:56 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 14 00 c0 08 81 bd 69 51 00 00 00 04 00 00 00 00 + HSP_SEL_TYPE_SOC_SVN_READ_FROM_IMAGE -> SVN + Firmware Value=0x00 +0x15 2026-03-20 17:16:56 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 15 00 c0 08 81 bd 69 51 00 00 00 0e 0e 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MESH_CONFIG +0x16 2026-03-20 17:16:56 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 16 00 c0 08 81 bd 69 51 00 00 00 0d 0e 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MESH_CONFIG +0x17 2026-03-20 17:16:56 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 17 00 c0 08 81 bd 69 51 00 00 00 0e 0b 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_DDR_PHY +0x18 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 18 00 c0 09 81 bd 69 51 00 00 00 0d 0b 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_DDR_PHY +0x19 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 19 00 c0 09 81 bd 69 51 00 00 00 0e 09 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_FUSE_OVERRIDE_DIE_0 +0x1A 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 1a 00 c0 09 81 bd 69 51 00 00 00 0d 09 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_FUSE_OVERRIDE_DIE_0 +0x1B 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 1b 00 c0 09 81 bd 69 51 00 00 00 0e 0a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_FUSE_OVERRIDE_DIE_1 +0x1C 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 1c 00 c0 09 81 bd 69 51 00 00 00 0d 0a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_FUSE_OVERRIDE_DIE_1 +0x1D 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 1d 00 c0 09 81 bd 69 51 00 00 00 0e 07 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SCP +0x1E 2026-03-20 17:16:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 1e 00 c0 09 81 bd 69 51 00 00 00 0d 07 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SCP +0x1F 2026-03-20 17:16:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 1f 00 c0 0a 81 bd 69 51 00 00 00 11 06 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_RUN_IFT_NOT_ENABLE +0x20 2026-03-20 17:16:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 20 00 c0 0a 81 bd 69 51 00 00 01 08 00 00 00 00 + HSP_SEL_TYPE_ACTIVE_BOOT_SLOT -> ActiveBootSlotA +0x21 2026-03-20 17:16:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 21 00 c0 0a 81 bd 69 51 00 00 01 11 06 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_RUN_IFT_NOT_ENABLE +0x22 2026-03-20 17:18:35 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 22 00 c0 6b 81 bd 69 51 00 00 01 11 08 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_DDR_INIT_DONE +0x23 2026-03-20 17:18:36 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 23 00 c0 6c 81 bd 69 51 00 00 00 11 08 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_DDR_INIT_DONE +0x24 2026-03-20 17:18:53 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 24 00 c0 7d 81 bd 69 51 00 00 00 11 09 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_COPYING_FW_TO_DDR_COMPLETE +0x25 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 25 00 c0 81 81 bd 69 51 00 00 00 0e 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x26 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 26 00 c0 81 81 bd 69 51 00 00 01 04 00 00 00 00 + HSP_SEL_TYPE_SOC_SVN_READ_FROM_IMAGE -> SVN + Firmware Value=0x00 +0x27 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 27 00 c0 81 81 bd 69 51 00 00 01 04 00 00 00 00 + HSP_SEL_TYPE_SOC_SVN_READ_FROM_IMAGE -> SVN + Firmware Value=0x00 +0x28 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 28 00 c0 81 81 bd 69 51 00 00 01 0e 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x29 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 29 00 c0 81 81 bd 69 51 00 00 00 0d 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x2A 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 2a 00 c0 81 81 bd 69 51 00 00 01 0d 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x2B 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 2b 00 c0 81 81 bd 69 51 00 00 00 0e 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PCIE_PHY +0x2C 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 2c 00 c0 81 81 bd 69 51 00 00 01 0e 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PCIE_PHY +0x2D 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 2d 00 c0 81 81 bd 69 51 00 00 01 0d 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PCIE_PHY +0x2E 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 2e 00 c0 81 81 bd 69 51 00 00 01 0e 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MCP +0x2F 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 2f 00 c0 81 81 bd 69 51 00 00 00 0d 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PCIE_PHY +0x30 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 30 00 c0 81 81 bd 69 51 00 00 00 0e 1e 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MSCP_TRACE +0x31 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 31 00 c0 81 81 bd 69 51 00 00 01 0d 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MCP +0x32 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 32 00 c0 81 81 bd 69 51 00 00 00 0d 1e 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MSCP_TRACE +0x33 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 33 00 c0 81 81 bd 69 51 00 00 00 0e 27 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_ACCEL_MANIFEST +0x34 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 34 00 c0 81 81 bd 69 51 00 00 01 0e 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_ITCM +0x35 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 35 00 c0 81 81 bd 69 51 00 00 00 0d 27 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_ACCEL_MANIFEST +0x36 2026-03-20 17:18:57 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 36 00 c0 81 81 bd 69 51 00 00 00 0e 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MCP +0x37 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 37 00 c0 82 81 bd 69 51 00 00 01 0d 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_ITCM +0x38 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 38 00 c0 82 81 bd 69 51 00 00 01 0e 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_DTCM +0x39 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 39 00 c0 82 81 bd 69 51 00 00 00 0d 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MCP +0x3A 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 3a 00 c0 82 81 bd 69 51 00 00 00 0e 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_ITCM +0x3B 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 3b 00 c0 82 81 bd 69 51 00 00 01 0d 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_DTCM +0x3C 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 3c 00 c0 82 81 bd 69 51 00 00 01 0e 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_ITCM +0x3D 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 3d 00 c0 82 81 bd 69 51 00 00 01 0d 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_ITCM +0x3E 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 3e 00 c0 82 81 bd 69 51 00 00 01 0e 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_DTCM +0x3F 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 3f 00 c0 82 81 bd 69 51 00 00 00 0d 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_ITCM +0x40 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 40 00 c0 82 81 bd 69 51 00 00 01 0d 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_DTCM +0x41 2026-03-20 17:18:58 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 41 00 c0 82 81 bd 69 51 00 00 00 0e 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_DTCM +0x42 2026-03-20 17:18:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 42 00 c0 83 81 bd 69 51 00 00 00 0d 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_DTCM +0x43 2026-03-20 17:18:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 43 00 c0 83 81 bd 69 51 00 00 00 0e 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_ITCM +0x44 2026-03-20 17:18:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 44 00 c0 83 81 bd 69 51 00 00 00 0d 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_ITCM +0x45 2026-03-20 17:18:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 45 00 c0 83 81 bd 69 51 00 00 00 0e 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_DTCM +0x46 2026-03-20 17:18:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 46 00 c0 83 81 bd 69 51 00 00 00 0d 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_DTCM +0x47 2026-03-20 17:19:47 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 47 00 c0 b3 81 bd 69 51 00 00 00 0e 11 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_SECURITY_MONITOR +0x48 2026-03-20 17:19:47 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 48 00 c0 b3 81 bd 69 51 00 00 00 0d 11 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_SECURITY_MONITOR +0x49 2026-03-20 17:19:47 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 49 00 c0 b3 81 bd 69 51 00 00 00 0e 12 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_MANAGEMENT_MODE +0x4A 2026-03-20 17:19:49 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 4a 00 c0 b5 81 bd 69 51 00 00 00 0d 12 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_MANAGEMENT_MODE +0x4B 2026-03-20 17:19:49 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 4b 00 c0 b5 81 bd 69 51 00 00 00 0e 13 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_UEFI +0x4C 2026-03-20 17:19:50 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 4c 00 c0 b6 81 bd 69 51 00 00 00 0d 13 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_UEFI +0x4D 2026-03-20 17:19:50 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 4d 00 c0 b6 81 bd 69 51 00 00 00 0e 14 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_SPM +0x4E 2026-03-20 17:19:50 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 4e 00 c0 b6 81 bd 69 51 00 00 00 0d 14 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_SPM +0x4F 2026-03-20 17:19:50 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 4f 00 c0 b6 81 bd 69 51 00 00 00 0e 15 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_RMM +0x50 2026-03-20 17:19:51 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 50 00 c0 b7 81 bd 69 51 00 00 00 0d 15 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_RMM +0x51 2026-03-20 17:19:51 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 51 00 c0 b7 81 bd 69 51 00 00 00 0e 26 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_RP_DATA +0x52 2026-03-20 17:19:51 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 52 00 c0 b7 81 bd 69 51 00 00 00 0d 26 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_RP_DATA +0x53 2026-03-20 17:19:52 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 53 00 c0 b8 81 bd 69 51 00 00 00 0e 25 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_RP_EXE +0x54 2026-03-20 17:19:52 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 54 00 c0 b8 81 bd 69 51 00 00 00 0d 25 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_RP_EXE +0x55 2026-03-20 17:19:52 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 55 00 c0 b8 81 bd 69 51 00 00 00 11 0a 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_SECONDARY_DIE_MSCP_BOOT_COMPLETE_TIMER_ + DISARMED +0x56 2026-03-20 17:19:54 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 56 00 c0 ba 81 bd 69 51 00 00 00 00 01 00 00 00 + HSP_SEL_TYPE_PROFILE_SELECTION -> + ProfileValue=0x01 +0x57 2026-03-20 17:19:54 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 57 00 c0 ba 81 bd 69 51 00 00 00 00 01 00 00 00 + HSP_SEL_TYPE_PROFILE_SELECTION -> + ProfileValue=0x01 +0x58 2026-03-20 17:20:28 Platform System Event(UEFI)=0xCA N/A N/A Information -> Ready To Boot Event N/A OEM Timestamped 58 00 ca dc 81 bd 69 00 00 2c 00 03 00 00 ff ff +0x59 2026-03-20 17:20:35 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 59 00 c0 e3 81 bd 69 51 00 00 00 11 0b 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_RECV_UEFI_BOOT_COMPLETED_NOTIFY +0x5A 2026-03-20 17:20:35 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 5a 00 c0 e3 81 bd 69 51 00 00 00 0f 00 00 00 00 + HSP_SEL_TYPE_FIRMWARE_SVN_UPDATE_STATE -> + HSP_SEL_FIRMWARE_SVN_UPDATED_SUCCESS +0x5B 2026-03-20 17:20:37 Platform System Event(UEFI)=0xCA N/A N/A Information -> PXE Start N/A OEM Timestamped 5b 00 ca e5 81 bd 69 00 00 2c 00 00 20 00 ff ff +0x5C 2026-03-20 17:20:37 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 5c 00 c0 e5 81 bd 69 51 00 00 01 0f 00 00 00 00 + HSP_SEL_TYPE_FIRMWARE_SVN_UPDATE_STATE -> + HSP_SEL_FIRMWARE_SVN_UPDATED_SUCCESS +0x5D 2026-03-20 17:21:36 Platform System Event(UEFI)=0xCA N/A N/A Error: Ipv4 PXE error code -> Server response N/A OEM Timestamped 5d 00 ca 20 82 bd 69 00 00 2c 02 01 20 18 ff ff + timeout +0x5E 2026-03-20 17:21:36 Platform System Event(UEFI)=0xCA N/A N/A Information -> PXE Start N/A OEM Timestamped 5e 00 ca 20 82 bd 69 00 00 2c 00 00 20 00 ff ff +0x5F 2026-03-20 17:22:40 Platform System Event(UEFI)=0xCA N/A N/A Error: Ipv6 PXE error code -> No valid offer N/A OEM Timestamped 5f 00 ca 60 82 bd 69 00 00 2c 02 02 20 16 ff ff + received +0x60 2026-03-20 17:22:41 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 60 00 c0 61 82 bd 69 51 00 00 00 11 0b 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_RECV_UEFI_BOOT_COMPLETED_NOTIFY +0x61 2026-03-20 17:23:31 Platform System Event(UEFI)=0xCA N/A N/A Information -> Exit Boot Service Event N/A OEM Timestamped 61 00 ca 93 82 bd 69 00 00 2c 00 02 00 00 ff ff +0x62 2026-03-20 17:24:33 Windows Event(Server Mgmt. Software)=0x00 Base OS Boot/Installation Status Sensor-Specific Discrete Base OS Boot/Installation Status: C: boot ASSERTED Composite 62 00 02 d1 82 bd 69 41 00 04 1f 00 6f 01 ff ff + completed, Boot Timestamp: Fri, Mar 20, 2026 63 00 dc d2 82 bd 69 37 01 00 00 a1 82 bd 69 00 + 05:23:45 PM (Relative to Windows Timezone) +0x64 2026-03-20 17:24:34 Generic Purpose(BMC)=0xD9 N/A N/A Trust Mode Change -> Current Trust Mode: General N/A OEM Timestamped 64 00 d9 d2 82 bd 69 20 00 00 04 01 01 01 02 00 + Purpose, New Trust Mode: General Purpose, Change + Method: Host OS Boot-up, Trust Mode Type: + Immediate, Trust JSON File: Available +0x65 2026-03-20 18:00:43 BMC_Health(BMC)=0xEF OEM Management Subsystem Health OEM Discrete Firmware Boot SPI Flash: SPI Flash ID=Primary, ASSERTED System Event 65 00 02 4b 8b bd 69 20 00 04 e0 ef 70 a2 01 00 + Non PMDU SKU +0x66 2026-03-20 18:00:45 FailSafe_Status(BMC)=0xFC OEM Sensor-Specific Discrete Failsafe Status: Normal ASSERTED System Event 66 00 02 4d 8b bd 69 20 00 04 e2 fc 6f 00 ff ff +0x67 2026-03-20 18:01:01 Generic Purpose(BMC)=0xD9 N/A N/A Manticore -> Keep Alive Normal N/A OEM Timestamped 67 00 d9 5d 8b bd 69 20 00 00 01 01 ff ff ff ff +0x68 2026-03-20 18:01:08 Power_Status(BMC)=0x97 System ACPI Power State Sensor-Specific Discrete SOC Power State: S5/G2 "Soft Off" -> SLEEP_S5_N ASSERTED System Event 68 00 02 64 8b bd 69 20 00 04 22 97 6f 05 ff ff + Asserted by SoC +0x69 2026-03-20 18:01:08 Power_Status(BMC)=0x97 System ACPI Power State Sensor-Specific Discrete SOC Power State: S0/G0 "Working" -> SOCPWRGOOD ASSERTED System Event 69 00 02 64 8b bd 69 20 00 04 22 97 6f 00 ff ff + High +0x6A 2026-03-20 18:01:09 BMC_Health(BMC)=0xEF OEM Management Subsystem Health OEM Discrete BMC Reset -> Virtual Reseat, Trigger Source: ASSERTED System Event 6a 00 02 65 8b bd 69 20 00 04 e0 ef 70 a8 03 01 + RM/RSCM +0x6B 2026-03-20 18:01:10 SCM_Secure_CPLD(BMC)=0xF6 BMC OEM OEM Discrete CPLD Status Code -> BMC Boot CPLD Status: Secure ASSERTED System Event 6b 00 02 66 8b bd 69 20 00 04 db f6 70 a5 00 ff + Boot is ENABLED, Public Key Area LOCKED, Feature + Row (Anti-Rollback) ENABLED, CFG1 Area LOCKED, + AUTH_DONE Bit SET for CFG0, AUTH_DONE Bit SET for + CFG1, Secure Boot Enabled CPLD FW is PROGRAMMED + for CFG0, Secure Boot Enabled CPLD FW is + PROGRAMMED for CFG1 +0x6C 2026-03-20 18:01:10 SCM_Secure_CPLD(BMC)=0xF6 BMC OEM OEM Discrete CPLD Status Code -> SCM CPLD Booted from CFG0 ASSERTED System Event 6c 00 02 66 8b bd 69 20 00 04 db f6 70 a6 00 ff +0x6D 2026-03-20 18:01:11 Generic Purpose(BMC)=0xD9 N/A N/A Trust Mode Change -> Current Trust Mode: General N/A OEM Timestamped 6d 00 d9 67 8b bd 69 20 00 00 04 01 01 03 02 00 + Purpose, New Trust Mode: General Purpose, Change + Method: BMC Boot-up, Trust Mode Type: Immediate, + Trust JSON File: Available +0x6E 2026-03-20 18:01:12 HPM_Secure_CPLD(BMC)=0xF7 BMC OEM OEM Discrete CPLD Status Code -> BMC Boot CPLD Status: Secure ASSERTED System Event 6e 00 02 68 8b bd 69 20 00 04 db f7 70 a5 00 ff + Boot is ENABLED, Public Key Area LOCKED, Feature + Row (Anti-Rollback) ENABLED, CFG1 Area LOCKED, + AUTH_DONE Bit SET for CFG0, AUTH_DONE Bit SET for + CFG1, Secure Boot Enabled CPLD FW is PROGRAMMED + for CFG0, Secure Boot Enabled CPLD FW is + PROGRAMMED for CFG1 +0x6F 2026-03-20 18:01:12 HPM_Secure_CPLD(BMC)=0xF7 BMC OEM OEM Discrete CPLD Status Code -> HPM CPLD Booted from CFG0 ASSERTED System Event 6f 00 02 68 8b bd 69 20 00 04 db f7 70 a6 00 ff +0x70 2026-03-20 18:01:15 System Event: Timestamp Clock Synch(BMC)=0x07 System Event Sensor-Specific Discrete SEL Timestamp Clock Updated ASSERTED System Event 70 00 02 6b 8b bd 69 20 00 04 12 07 6f c5 80 ff +0x71 2026-03-20 18:02:47 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 71 00 c0 c7 8b bd 69 51 00 00 01 11 08 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_DDR_INIT_DONE +0x72 2026-03-20 18:02:48 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 72 00 c0 c8 8b bd 69 51 00 00 00 11 08 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_DDR_INIT_DONE +0x73 2026-03-20 18:03:05 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 73 00 c0 d9 8b bd 69 51 00 00 00 11 09 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_COPYING_FW_TO_DDR_COMPLETE +0x74 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 74 00 c0 dd 8b bd 69 51 00 00 00 0e 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x75 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 75 00 c0 dd 8b bd 69 51 00 00 01 04 00 00 00 00 + HSP_SEL_TYPE_SOC_SVN_READ_FROM_IMAGE -> SVN + Firmware Value=0x00 +0x76 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 76 00 c0 dd 8b bd 69 51 00 00 01 04 00 00 00 00 + HSP_SEL_TYPE_SOC_SVN_READ_FROM_IMAGE -> SVN + Firmware Value=0x00 +0x77 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 77 00 c0 dd 8b bd 69 51 00 00 01 0e 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x78 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 78 00 c0 dd 8b bd 69 51 00 00 00 0d 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x79 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 79 00 c0 dd 8b bd 69 51 00 00 01 0d 29 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_HSP_TELEMETRY +0x7A 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 7a 00 c0 dd 8b bd 69 51 00 00 00 0e 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PCIE_PHY +0x7B 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 7b 00 c0 dd 8b bd 69 51 00 00 01 0e 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PCIE_PHY +0x7C 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 7c 00 c0 dd 8b bd 69 51 00 00 01 0d 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PCIE_PHY +0x7D 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 7d 00 c0 dd 8b bd 69 51 00 00 01 0e 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MCP +0x7E 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 7e 00 c0 dd 8b bd 69 51 00 00 00 0d 0f 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PCIE_PHY +0x7F 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 7f 00 c0 dd 8b bd 69 51 00 00 00 0e 1e 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MSCP_TRACE +0x80 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 80 00 c0 dd 8b bd 69 51 00 00 01 0d 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MCP +0x81 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 81 00 c0 dd 8b bd 69 51 00 00 00 0d 1e 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MSCP_TRACE +0x82 2026-03-20 18:03:09 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 82 00 c0 dd 8b bd 69 51 00 00 00 0e 27 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_ACCEL_MANIFEST +0x83 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 83 00 c0 de 8b bd 69 51 00 00 01 0e 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_ITCM +0x84 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 84 00 c0 de 8b bd 69 51 00 00 00 0d 27 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_ACCEL_MANIFEST +0x85 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 85 00 c0 de 8b bd 69 51 00 00 00 0e 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_MCP +0x86 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 86 00 c0 de 8b bd 69 51 00 00 01 0d 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_ITCM +0x87 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 87 00 c0 de 8b bd 69 51 00 00 01 0e 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_DTCM +0x88 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 88 00 c0 de 8b bd 69 51 00 00 01 0d 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_DTCM +0x89 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 89 00 c0 de 8b bd 69 51 00 00 00 0d 08 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_MCP +0x8A 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 8a 00 c0 de 8b bd 69 51 00 00 00 0e 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_ITCM +0x8B 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 8b 00 c0 de 8b bd 69 51 00 00 01 0e 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_ITCM +0x8C 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 8c 00 c0 de 8b bd 69 51 00 00 01 0d 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_ITCM +0x8D 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 8d 00 c0 de 8b bd 69 51 00 00 01 0e 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_DTCM +0x8E 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped 8e 00 c0 de 8b bd 69 51 00 00 01 0d 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_DTCM +0x8F 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 8f 00 c0 de 8b bd 69 51 00 00 00 0d 18 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_ITCM +0x90 2026-03-20 18:03:10 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 90 00 c0 de 8b bd 69 51 00 00 00 0e 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_SDM_DTCM +0x91 2026-03-20 18:03:11 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 91 00 c0 df 8b bd 69 51 00 00 00 0d 17 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_SDM_DTCM +0x92 2026-03-20 18:03:11 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 92 00 c0 df 8b bd 69 51 00 00 00 0e 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_ITCM +0x93 2026-03-20 18:03:11 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 93 00 c0 df 8b bd 69 51 00 00 00 0d 1a 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_ITCM +0x94 2026-03-20 18:03:11 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 94 00 c0 df 8b bd 69 51 00 00 00 0e 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_CDED_DTCM +0x95 2026-03-20 18:03:12 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 95 00 c0 e0 8b bd 69 51 00 00 00 0d 19 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_CDED_DTCM +0x96 2026-03-20 18:03:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 96 00 c0 0f 8c bd 69 51 00 00 00 0e 11 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_SECURITY_MONITOR +0x97 2026-03-20 18:03:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 97 00 c0 0f 8c bd 69 51 00 00 00 0d 11 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_SECURITY_MONITOR +0x98 2026-03-20 18:03:59 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 98 00 c0 0f 8c bd 69 51 00 00 00 0e 12 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_MANAGEMENT_MODE +0x99 2026-03-20 18:04:01 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 99 00 c0 11 8c bd 69 51 00 00 00 0d 12 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_MANAGEMENT_MODE +0x9A 2026-03-20 18:04:01 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 9a 00 c0 11 8c bd 69 51 00 00 00 0e 13 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_UEFI +0x9B 2026-03-20 18:04:02 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 9b 00 c0 12 8c bd 69 51 00 00 00 0d 13 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_UEFI +0x9C 2026-03-20 18:04:02 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 9c 00 c0 12 8c bd 69 51 00 00 00 0e 14 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_PE_SPM +0x9D 2026-03-20 18:04:02 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 9d 00 c0 12 8c bd 69 51 00 00 00 0d 14 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_PE_SPM +0x9E 2026-03-20 18:04:02 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 9e 00 c0 12 8c bd 69 51 00 00 00 0e 15 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_RMM +0x9F 2026-03-20 18:04:03 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped 9f 00 c0 13 8c bd 69 51 00 00 00 0d 15 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_RMM +0xA0 2026-03-20 18:04:03 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a0 00 c0 13 8c bd 69 51 00 00 00 0e 26 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_RP_DATA +0xA1 2026-03-20 18:04:04 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a1 00 c0 14 8c bd 69 51 00 00 00 0d 26 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_RP_DATA +0xA2 2026-03-20 18:04:04 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a2 00 c0 14 8c bd 69 51 00 00 00 0e 25 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_PROGRESS -> + HSP_FIRMWARE_ID_RP_EXE +0xA3 2026-03-20 18:04:04 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a3 00 c0 14 8c bd 69 51 00 00 00 0d 25 00 00 00 + HSP_SEL_TYPE_FIRMWARE_LOAD_COMPLETED -> + HSP_FIRMWARE_ID_RP_EXE +0xA4 2026-03-20 18:04:04 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a4 00 c0 14 8c bd 69 51 00 00 00 11 0a 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_SECONDARY_DIE_MSCP_BOOT_COMPLETE_TIMER_ + DISARMED +0xA5 2026-03-20 18:04:06 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a5 00 c0 16 8c bd 69 51 00 00 00 00 01 00 00 00 + HSP_SEL_TYPE_PROFILE_SELECTION -> + ProfileValue=0x01 +0xA6 2026-03-20 18:04:06 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a6 00 c0 16 8c bd 69 51 00 00 00 00 01 00 00 00 + HSP_SEL_TYPE_PROFILE_SELECTION -> + ProfileValue=0x01 +0xA7 2026-03-20 18:04:39 Platform System Event(UEFI)=0xCA N/A N/A Information -> Ready To Boot Event N/A OEM Timestamped a7 00 ca 37 8c bd 69 00 00 2c 00 03 00 00 ff ff +0xA8 2026-03-20 18:04:46 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a8 00 c0 3e 8c bd 69 51 00 00 00 11 0b 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_RECV_UEFI_BOOT_COMPLETED_NOTIFY +0xA9 2026-03-20 18:04:46 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped a9 00 c0 3e 8c bd 69 51 00 00 00 0f 00 00 00 00 + HSP_SEL_TYPE_FIRMWARE_SVN_UPDATE_STATE -> + HSP_SEL_FIRMWARE_SVN_UPDATED_SUCCESS +0xAA 2026-03-20 18:04:46 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 1, HSP SEL Type: N/A OEM Timestamped aa 00 c0 3e 8c bd 69 51 00 00 01 0f 00 00 00 00 + HSP_SEL_TYPE_FIRMWARE_SVN_UPDATE_STATE -> + HSP_SEL_FIRMWARE_SVN_UPDATED_SUCCESS +0xAB 2026-03-20 18:04:48 Platform System Event(UEFI)=0xCA N/A N/A Information -> PXE Start N/A OEM Timestamped ab 00 ca 40 8c bd 69 00 00 2c 00 00 20 00 ff ff +0xAC 2026-03-20 18:05:47 Platform System Event(UEFI)=0xCA N/A N/A Error: Ipv4 PXE error code -> Server response N/A OEM Timestamped ac 00 ca 7b 8c bd 69 00 00 2c 02 01 20 18 ff ff + timeout +0xAD 2026-03-20 18:05:47 Platform System Event(UEFI)=0xCA N/A N/A Information -> PXE Start N/A OEM Timestamped ad 00 ca 7b 8c bd 69 00 00 2c 00 00 20 00 ff ff +0xAE 2026-03-20 18:06:51 Platform System Event(UEFI)=0xCA N/A N/A Error: Ipv6 PXE error code -> No valid offer N/A OEM Timestamped ae 00 ca bb 8c bd 69 00 00 2c 02 02 20 16 ff ff + received +0xAF 2026-03-20 18:06:52 HSP SEL(HSP)=0xC0 N/A N/A HSP Die: 0, HSP SEL Type: N/A OEM Timestamped af 00 c0 bc 8c bd 69 51 00 00 00 11 0b 00 00 00 + HSP_SEL_TYPE_RESET_MANAGER_STATE -> + HSP_SEL_RM_RECV_UEFI_BOOT_COMPLETED_NOTIFY +0xB0 2026-03-20 18:07:42 Platform System Event(UEFI)=0xCA N/A N/A Information -> Exit Boot Service Event N/A OEM Timestamped b0 00 ca ee 8c bd 69 00 00 2c 00 02 00 00 ff ff +0xB1 2026-03-20 18:08:46 Windows Event(Server Mgmt. Software)=0x00 Base OS Boot/Installation Status Sensor-Specific Discrete Base OS Boot/Installation Status: C: boot ASSERTED Composite b1 00 02 2e 8d bd 69 41 00 04 1f 00 6f 01 ff ff + completed, Boot Timestamp: Fri, Mar 20, 2026 b2 00 dc 2e 8d bd 69 37 01 00 00 fc 8c bd 69 00 + 06:07:56 PM (Relative to Windows Timezone) +0xB3 2026-03-20 18:08:47 Generic Purpose(BMC)=0xD9 N/A N/A Trust Mode Change -> Current Trust Mode: General N/A OEM Timestamped b3 00 d9 2f 8d bd 69 20 00 00 04 01 01 01 02 00 + Purpose, New Trust Mode: General Purpose, Change + Method: Host OS Boot-up, Trust Mode Type: + Immediate, Trust JSON File: Available + +Decode Errors: +-------------- +Decode Error: Generic Purpose + - Attributes: + ID: 0x07 + Timestamp: Friday, March 20, 2026 05:16:42 PM GMT + Sensor/OEM(Owner): Generic Purpose(BMC)=0xD9 + Sensor Type: N/A + Event Type: N/A + Message: Unknown=8, [DECODE ERROR] + Record Type: OEM Timestamped + Raw Hex: 07 00 d9 fa 80 bd 69 20 00 00 08 00 01 ff ff ff + - Error: Invalid Component ID -> OEM Defined: Byte 11=0x08 + +Parse Errors: +------------- +Unable to parse line 1: "Last login: Fri Mar 20 18:20:31 2026 from 192.42.1.1" +Unable to parse line 2: "" +Unable to parse line 3: "show system log read -i 6" +Unable to parse line 4: "RScmCli# show system log read -i 6" +Unable to parse line 184: "" +Unable to parse line 185: "Completion Code: Success" +Unable to parse line 186: "RScmCli# " + +File '2026-03-20-18-20-56_selRecords_6.txt' contains 186 lines. Successfully parsed 179 SEL records. 1 Decode Error. 7 Parse Errors. diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_1.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_1.txt new file mode 100644 index 0000000000..bec723f6c9 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_1.txt @@ -0,0 +1,239 @@ +ipmiutil sel version 3.13 +-- BMC version 3.46, IPMI version 2.0 +SEL Size = 2046 records (Used=230, Free=1816) +RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail +0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared +0002 | 12/18/23 20:16:51 | INF | Sms| OS Critical Stop | na | OS Graceful Shutdown +0003 | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 00 01 00 02 80 00 +0004 | 12/18/23 20:16:51 | CRT | EFI | Critical Interrupt | #a1 | Bus Warn 6f [a7 00 c9] +0005 | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 02 u r +0006 | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 03 c +0007 | 12/18/23 20:16:51 | INF | dd | Critical Interrupt | #a1 | Bus Warn 6f [a7 00 4a] +0008 | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 05 p u +0009 | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 06 t e +000a | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 07 r +000b | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 08 w i +000c | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 09 l l +000d | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 0a r +000e | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 0b e s +000f | 12/18/23 20:16:51 | INF | dd | 000137 | Microsoft | OEM Event 0c t a +0010 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 0d r t +0011 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 0e a +0012 | 12/18/23 20:16:52 | CRT | BMC | Processor | #c1 | IERR 6f [00 ff ff] +0013 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 10 1 2 +0014 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 11 / 1 +0015 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 12 9 / +0016 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 13 2 0 +0017 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 14 2 3 +0018 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 14 2 3 +0019 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 16 : 1 +001a | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 17 4 : +001b | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 18 5 3 +001c | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 19 A +001d | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 1a M +001e | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 1b 2 3 +001f | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 1c c +0020 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 1d o m +0021 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 1e p l +0022 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 1f e t +0023 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 20 e +0024 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 21 2 3 +0025 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 22 e +0026 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 23 i n +0027 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 24 s t +0028 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 25 a l +0029 | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 26 l a +002a | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 27 t i +002b | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 28 o n +002c | 12/18/23 20:16:52 | INF | dd | 000137 | Microsoft | OEM Event 29 o +002d | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 2a f +002e | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 2b a p +002f | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 2c p l +0030 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 2d i c +0031 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 2e a t +0032 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 2f i o +0033 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 30 n s +0034 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 31 a +0035 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 32 n d +0036 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 33 s +0037 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 34 o f +0038 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 35 t w +0039 | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 36 a r +003a | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 37 e +003b | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 38 u p +003c | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 39 2 3 +003d | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 3a t e +003e | 12/18/23 20:16:53 | INF | dd | 000137 | Microsoft | OEM Event 3b s . +003f | 12/18/23 20:17:07 | INF | c8 | 009c9c | | OEM Event 00 02 00 00 00 00 +0040 | 12/18/23 20:17:07 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +0041 | 12/18/23 20:17:15 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +0042 | 12/18/23 20:17:40 | INF | BMC | OEM(e1) | BMC Health | - +0043 | 12/18/23 20:17:40 | INF | BMC | OEM(e1) | BMC Health | - +0044 | 12/18/23 20:17:40 | INF | BMC | OEM(e1) | BMC Health | - +0045 | 12/18/23 20:17:40 | INF | BMC | OEM(e1) | BMC Health | - +0046 | 12/18/23 20:18:41 | INF | BMC | OEM(e1) | BMC Health | - +0047 | 12/18/23 20:18:41 | INF | BMC | OEM(e1) | BMC Health | - +0048 | 12/18/23 20:18:41 | INF | BMC | OEM(e1) | BMC Health | - +0049 | 12/18/23 20:18:41 | INF | BMC | OEM(e1) | BMC Health | - +004a | 12/18/23 20:19:30 | INF | Sms | OS Boot | na | C: boot completed +004b | 12/18/23 20:19:30 | INF | dc | 000137 | Microsoft | OEM Event 00 b0 19 81 65 00 +004c | 12/20/23 01:18:16 | INF | Sms | OS Critical Stop | na | OS Graceful Shutdown +004d | 12/20/23 01:18:16 | INF | dd | 000137 | Microsoft | OEM Event 00 01 00 04 84 00 +004e | 12/20/23 01:18:23 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +004f | 12/20/23 01:18:23 | INF | c8 | 009c9c | | OEM Event 00 02 00 00 00 00 +0050 | 12/20/23 01:18:31 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +0051 | 12/20/23 01:18:56 | INF | BMC | OEM(e1) | BMC Health | - +0052 | 12/20/23 01:18:56 | INF | BMC | OEM(e1) | BMC Health | - +0053 | 12/20/23 01:18:56 | INF | BMC | OEM(e1) | BMC Health | - +0054 | 12/20/23 01:18:56 | INF | BMC | OEM(e1) | BMC Health | - +0055 | 12/20/23 01:19:57 | INF | BMC | OEM(e1) | BMC Health | - +0056 | 12/20/23 01:19:57 | INF | BMC | OEM(e1) | BMC Health | - +0057 | 12/20/23 01:19:57 | INF | BMC | OEM(e1) | BMC Health | - +0058 | 12/20/23 01:19:58 | INF | BMC | OEM(e1) | BMC Health | - +0059 | 12/20/23 01:20:45 | INF | Sms | OS Boot | na | C: boot completed +005a | 12/20/23 01:20:45 | INF | dc | 000137 | Microsoft | OEM Event 00 cc b1 82 65 00 +005b | 01/04/24 00:23:28 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor af fault +005c | 01/04/24 00:24:18 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor af fault +005d | 01/15/24 21:36:30 | INF | Sms | OS Critical Stop | na | OS Graceful Shutdown +005e | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 00 01 00 02 80 00 +005f | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 01 Y o +0060 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 02 u r +0061 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 03 c +0062 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 04 o m +0063 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 05 p u +0064 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 06 t e +0065 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 07 r +0066 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 08 w i +0067 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 09 l l +0068 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 0a r +0069 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 0b e s +006a | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 0c t a +006b | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 0d r t +006c | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 0e a +006d | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 0f t +006e | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 10 1 / +006f | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 11 1 6 +0070 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 12 / 2 +0071 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 13 0 2 +0072 | 01/15/24 21:36:30 | INF | dd | 000137 | Microsoft | OEM Event 14 4 +0073 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 15 5 : +0074 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 16 3 4 +0075 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 17 : 1 +0076 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 18 3 +0077 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 19 A M +0078 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 1a t +0079 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 1b o +007a | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 1c c o +007b | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 1d m p +007c | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 1e l e +007d | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 1f t e +007e | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 20 t +007f | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 21 h e +0080 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 22 i +0081 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 23 n s +0082 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 24 t a +0083 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 25 l l +0084 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 26 a t +0085 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 27 i o +0086 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 28 n +0087 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 29 o f +0088 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 2a a +0089 | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 2b p p +008a | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 2c l i +008b | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 2d c a +008c | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 2e t i +008d | 01/15/24 21:36:31 | INF | dd | 000137 | Microsoft | OEM Event 2f o n +008e | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 30 s +008f | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 31 a n +0090 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 32 d +0091 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 33 s o +0092 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 34 f t +0093 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 35 w a +0094 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 36 r e +0095 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 37 u +0096 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 38 p d +0097 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 39 a t +0098 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 3a e s +0099 | 01/15/24 21:36:32 | INF | dd | 000137 | Microsoft | OEM Event 3b . +009a | 01/15/24 21:36:47 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +009b | 01/15/24 21:36:47 | INF | c8 | 009c9c | | OEM Event 00 02 00 00 00 00 +009c | 01/15/24 21:36:55 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +009d | 01/15/24 21:37:32 | INF | BMC | OEM(e1) | BMC Health | - +009e | 01/15/24 21:37:32 | INF | BMC | OEM(e1) | BMC Health | - +009f | 01/15/24 21:37:33 | INF | BMC | OEM(e1) | BMC Health | - +00a0 | 01/15/24 21:37:33 | INF | BMC | OEM(e1) | BMC Health | - +00a1 | 01/15/24 21:38:33 | INF | BMC | OEM(e1) | BMC Health | - +00a2 | 01/15/24 21:38:34 | INF | BMC | OEM(e1) | BMC Health | - +00a3 | 01/15/24 21:38:34 | INF | BMC | OEM(e1) | BMC Health | - +00a4 | 01/15/24 21:38:34 | INF | BMC | OEM(e1) | BMC Health | - +00a5 | 01/15/24 21:39:27 | INF | Sms | OS Boot | na | C: boot completed +00a6 | 01/15/24 21:39:28 | INF | dc | 000137 | Microsoft | OEM Event 00 6f 16 a6 65 00 +00a7 | 01/19/24 07:18:01 | INF | Sms | OS Critical Stop | na | OS Graceful Shutdown +00a8 | 01/19/24 07:18:01 | INF | dd | 000137 | Microsoft | OEM Event 00 15 00 07 80 00 +00a9 | 01/19/24 07:18:07 | INF | BMC | OEM(d0) | NVDIMM | - +00aa | 01/19/24 07:18:07 | INF | BMC | OEM(d0) | NVDIMM | - +00ab | 01/19/24 07:18:07 | INF | BMC | ACPI Power State | ACPI State | S5/G2 soft-off +00ac | 01/19/24 08:48:37 | MAJ | BMC | Power Supply | PSU_Status_Event | Failure detected +00ad | 01/19/24 08:48:37 | CRT | EFI | Power Supply | #b3 | Predictive failure 6f [02 02 18] +00ae | 01/19/24 08:48:37 | INF | BMC | Power Supply | PSU_Status_Event | - +00af | 01/19/24 08:48:38 | INF | BMC | OEM(d0) | NVDIMM | - +00b0 | 01/19/24 08:48:38 | INF | BMC | Power Supply | PSU_Status_Event | - +00b1 | 01/22/24 00:03:50 | INF | BMC | OEM(e0) | BMC Health | - +00b2 | 01/22/24 00:03:50 | INF | BMC | Power Supply | PSU_Status_Event | Inserted +00b3 | 01/22/24 00:03:50 | INF | BMC | Battery | BAT_Status_Event | Present +00b4 | 01/22/24 00:03:52 | INF | BMC | Power Supply | PSU_Status_Event | - +00b5 | 01/22/24 00:03:52 | INF | BMC | Power Supply | PSU_Status_Event | AC Lost +00b6 | 01/22/24 00:03:52 | INF | BMC | Power Supply | PSU_Status_Event | - +00b7 | 01/22/24 00:03:53 | INF | BMC | Power Supply | PSU_Status_Event | - +00b8 | 01/22/24 00:04:18 | INF | BMC | ACPI Power State | ACPI State | S0/G0 Working +00b9 | 01/22/24 00:04:21 | INF | BMC | Watchdog_2 | na | - +00ba | 01/22/24 00:04:22 | INF | c8 | 009c9c | | OEM Event 00 02 00 00 00 00 +00bb | 01/22/24 00:04:30 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +00bc | 01/22/24 00:04:52 | INF | BMC | OEM(e1) | BMC Health | - +00bd | 01/22/24 00:04:52 | INF | BMC | OEM(e1) | BMC Health | - +00be | 01/22/24 00:04:52 | INF | BMC | OEM(e1) | BMC Health | - +00bf | 01/22/24 00:04:52 | INF | BMC | OEM(e1) | BMC Health | - +00c0 | 01/22/24 00:04:58 | INF | BMC | OEM(e1) | BMC Health | - +00c1 | 01/22/24 00:04:58 | INF | BMC | OEM(e1) | BMC Health | - +00c2 | 01/22/24 00:04:58 | INF | BMC | OEM(e1) | BMC Health | - +00c3 | 01/22/24 00:04:59 | INF | BMC | OEM(e1) | BMC Health | - +00c4 | 01/22/24 00:05:48 | INF | Sms | OS Boot | na | C: boot completed +00c5 | 01/22/24 00:05:48 | INF | dc | 000137 | Microsoft | OEM Event 00 b4 21 ae 65 00 +00c6 | 02/08/24 00:08:12 | INF | BMC | OEM(e0) | BMC Health | - +00c7 | 02/15/24 21:15:55 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor af fault +00c8 | 02/15/24 21:16:45 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor af fault +00c9 | 02/19/24 18:35:35 | INF | Sms | OS Critical Stop | na | OS Graceful Shutdown +00ca | 02/19/24 18:35:35 | INF | dd | 000137 | Microsoft | OEM Event 00 ff 00 00 80 00 +00cb | 02/19/24 18:35:50 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +00cc | 02/19/24 18:35:50 | INF | c8 | 009c9c | | OEM Event 00 02 00 00 00 00 +00cd | 02/19/24 18:35:59 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +00ce | 02/19/24 18:36:23 | INF | BMC | OEM(e1) | BMC Health | - +00cf | 02/19/24 18:36:23 | INF | BMC | OEM(e1) | BMC Health | - +00d0 | 02/19/24 18:36:24 | INF | BMC | OEM(e1) | BMC Health | - +00d1 | 02/19/24 18:36:24 | INF | BMC | OEM(e1) | BMC Health | - +00d2 | 02/19/24 18:37:24 | INF | BMC | OEM(e1) | BMC Health | - +00d3 | 02/19/24 18:37:24 | INF | BMC | OEM(e1) | BMC Health | - +00d4 | 02/19/24 18:37:25 | INF | BMC | OEM(e1) | BMC Health | - +00d5 | 02/19/24 18:37:25 | INF | BMC | OEM(e1) | BMC Health | - +00d6 | 02/19/24 18:38:19 | INF | Sms | OS Boot | na | C: boot completed +00d7 | 02/19/24 18:38:19 | INF | dc | 000137 | Microsoft | OEM Event 00 78 10 d4 65 00 +00d8 | 02/19/24 18:39:03 | INF | Sms | OS Critical Stop | na | OS Graceful Shutdown +00d9 | 02/19/24 18:39:03 | INF | dd | 000137 | Microsoft | OEM Event 00 03 00 02 80 00 +00da | 02/19/24 18:39:09 | INF | c8 | 009c9c | | OEM Event 00 02 00 00 00 00 +00db | 02/19/24 18:39:09 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +00dc | 02/19/24 18:39:16 | INF | BMC | Management Subsystem Health | Subsystem_Health | Sensor 35 fault +00dd | 02/19/24 18:39:41 | INF | BMC | OEM(e1) | BMC Health | - +00de | 02/19/24 18:39:42 | INF | BMC | OEM(e1) | BMC Health | - +00df | 02/19/24 18:39:42 | INF | BMC | OEM(e1) | BMC Health | - +00e0 | 02/19/24 18:39:42 | INF | BMC | OEM(e1) | BMC Health | - +00e1 | 02/19/24 18:40:42 | INF | BMC | OEM(e1) | BMC Health | - +00e2 | 02/19/24 18:40:43 | INF | BMC | OEM(e1) | BMC Health | - +00e3 | 02/19/24 18:40:43 | INF | BMC | OEM(e1) | BMC Health | - +00e4 | 02/19/24 18:40:43 | INF | BMC | OEM(e1) | BMC Health | - +00e5 | 02/19/24 18:41:35 | INF | Sms | OS Boot | na | C: boot completed +00e6 | 02/19/24 18:41:35 | INF | dc | 000137 | Microsoft | OEM Event 00 3d 11 d4 65 00 +00e7 | 02/19/24 18:42:00 | CRT | EFI| Critical Interrupt | #a1 | Fatal NMI 6f [ab 00 09] +00e8 | 02/19/24 18:42:00 | CRT | BMC | Processor | #8b | FRB2 timeout 6f [03 ff ff] +00e9 | 02/19/24 18:42:00 | CRT | BMC | Processor | #8b | FRB3 Proc Error 6f [04 ff ff] +00ea | 02/19/24 18:42:00 | CRT | EFI | Critical Interrupt | #a1 | Bus Warn 6f [a7 00 21] +ipmiutil sel, completed successfully diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_2.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_2.txt new file mode 100644 index 0000000000..09852e5bd4 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_decoded_example_2.txt @@ -0,0 +1,1946 @@ +ipmiutil sel version 3.18 +-- BMC version 18.11, IPMI version 2.0 +SEL Size = 2048 records (Used=1940, Free=3640) +Showing SEL Time as UTC +RecId | Date/Time | SEV | Src | Evt_Type | Sensor | Evt_detail +1601 | 01/28/25 04:03:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1602 | 01/28/25 04:03:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1603 | 01/28/25 04:03:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1604 | 01/28/25 04:03:51 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1605 | 01/28/25 04:03:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1606 | 01/28/25 04:03:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1607 | 01/28/25 04:03:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1608 | 01/28/25 04:03:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1609 | 01/28/25 04:03:51 | INF | BMC | OEM(e0) | BMC_Health | - +160a | 01/28/25 04:03:52 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=34.00 C, threshold=85.00 C +160b | 01/28/25 04:03:52 | INF | BMC | Fan | FANS_SWB_PWM | - +160c | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +160d | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +160e | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +160f | 01/28/25 04:03:52 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=34.00 C, threshold=80.00 C +1610 | 01/28/25 04:03:52 | INF | BMC | Fan | FANS_SWB_PWM | - +1611 | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1612 | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1613 | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1614 | 01/28/25 04:03:52 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=34.00 C, threshold=75.00 C +1615 | 01/28/25 04:03:52 | INF | BMC | Fan | FANS_SWB_PWM | - +1616 | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1617 | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1618 | 01/28/25 04:03:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1619 | 01/28/25 05:50:08 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +161a | 01/28/25 05:50:08 | INF | BMC | Fan | FANS_SWB_PWM | - +161b | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +161c | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +161d | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +161e | 01/28/25 05:50:09 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +161f | 01/28/25 05:50:09 | INF | BMC | Fan | FANS_SWB_PWM | - +1620 | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1621 | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1622 | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1623 | 01/28/25 05:50:09 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1624 | 01/28/25 05:50:09 | INF | BMC | Fan | FANS_SWB_PWM | - +1625 | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1626 | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1627 | 01/28/25 05:50:09 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1628 | 01/28/25 05:50:09 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=39.00 C, threshold=110.00 C +1629 | 01/28/25 05:50:09 | INF | BMC | Fan | FANS_SWB_PWM | - +162a | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +162b | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +162c | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +162d | 01/28/25 05:50:10 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=39.00 C, threshold=107.00 C +162e | 01/28/25 05:50:10 | INF | BMC | Fan | FANS_SWB_PWM | - +162f | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1630 | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1631 | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1632 | 01/28/25 05:50:10 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=39.00 C, threshold=105.00 C +1633 | 01/28/25 05:50:10 | INF | BMC | Fan | FANS_SWB_PWM | - +1634 | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1635 | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1636 | 01/28/25 05:50:10 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1637 | 01/28/25 08:09:43 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1638 | 01/28/25 08:09:43 | INF | BMC | Fan | FANS_SWB_PWM | - +1639 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +163a | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +163b | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +163c | 01/28/25 08:09:44 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +163d | 01/28/25 08:09:44 | INF | BMC | Fan | FANS_SWB_PWM | - +163e | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +163f | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1640 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1641 | 01/28/25 08:09:44 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1642 | 01/28/25 08:09:44 | INF | BMC | Fan | FANS_SWB_PWM | - +1643 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1644 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1645 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 96 94 96 ff ff +1646 | 01/28/25 08:09:44 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=34.00 C, threshold=85.00 C +1647 | 01/28/25 08:09:44 | INF | BMC | Fan | FANS_SWB_PWM | - +1648 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1649 | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +164a | 01/28/25 08:09:44 | INF | d0 | 000020 | | OEM Event 94 96 94 96 ff ff +164b | 01/28/25 08:09:45 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=34.00 C, threshold=80.00 C +164c | 01/28/25 08:09:45 | INF | BMC | Fan | FANS_SWB_PWM | - +164d | 01/28/25 08:09:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +164e | 01/28/25 08:09:45 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +164f | 01/28/25 08:09:45 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1650 | 01/28/25 08:09:45 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=34.00 C, threshold=75.00 C +1651 | 01/28/25 08:09:45 | INF | BMC | Fan | FANS_SWB_PWM | - +1652 | 01/28/25 08:09:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1653 | 01/28/25 08:09:45 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1654 | 01/28/25 08:09:45 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1655 | 01/29/25 04:20:23 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1656 | 01/29/25 04:20:23 | INF | BMC | Fan | FANS_SWB_PWM | - +1657 | 01/29/25 04:20:23 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1658 | 01/29/25 04:20:23 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +1659 | 01/29/25 04:20:23 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +165a | 01/29/25 04:20:23 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +165b | 01/29/25 04:20:23 | INF | BMC | Fan | FANS_SWB_PWM | - +165c | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +165d | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +165e | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +165f | 01/29/25 04:20:24 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1660 | 01/29/25 04:20:24 | INF | BMC | Fan | FANS_SWB_PWM | - +1661 | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1662 | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1663 | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1664 | 01/29/25 04:20:24 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1665 | 01/29/25 04:20:24 | INF | BMC | Fan | FANS_SWB_PWM | - +1666 | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1667 | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1668 | 01/29/25 04:20:24 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1669 | 01/29/25 04:20:24 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +166a | 01/29/25 04:20:24 | INF | BMC | Fan | FANS_SWB_PWM | - +166b | 01/29/25 04:20:25 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +166c | 01/29/25 04:20:25 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +166d | 01/29/25 04:20:25 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +166e | 01/29/25 04:20:25 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +166f | 01/29/25 04:20:25 | INF | BMC | Fan | FANS_SWB_PWM | - +1670 | 01/29/25 04:20:25 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1671 | 01/29/25 04:20:25 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1672 | 01/29/25 04:20:25 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1673 | 01/29/25 05:10:52 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1674 | 01/29/25 05:10:52 | INF | BMC | Fan | FANS_SWB_PWM | - +1675 | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1676 | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1677 | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1678 | 01/29/25 05:10:52 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1679 | 01/29/25 05:10:52 | INF | BMC | Fan | FANS_SWB_PWM | - +167a | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +167b | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +167c | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +167d | 01/29/25 05:10:52 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +167e | 01/29/25 05:10:52 | INF | BMC | Fan | FANS_SWB_PWM | - +167f | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1680 | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1681 | 01/29/25 05:10:52 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1682 | 01/29/25 05:10:53 | INF | BMC | OEM(e0) | BMC_Health | - +1683 | 01/29/25 05:10:53 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=38.00 C, threshold=110.00 C +1684 | 01/29/25 05:10:53 | INF | BMC | Fan | FANS_SWB_PWM | - +1685 | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1686 | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1687 | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1688 | 01/29/25 05:10:53 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=38.00 C, threshold=107.00 C +1689 | 01/29/25 05:10:53 | INF | BMC | Fan | FANS_SWB_PWM | - +168a | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +168b | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +168c | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +168d | 01/29/25 05:10:53 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=38.00 C, threshold=105.00 C +168e | 01/29/25 05:10:53 | INF | BMC | Fan | FANS_SWB_PWM | - +168f | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1690 | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1691 | 01/29/25 05:10:53 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1692 | 01/29/25 11:42:14 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1693 | 01/29/25 11:42:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1694 | 01/29/25 11:42:14 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1695 | 01/29/25 11:42:14 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +1696 | 01/29/25 11:42:14 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1697 | 01/29/25 11:42:14 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1698 | 01/29/25 11:42:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1699 | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +169a | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +169b | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 93 97 94 97 ff ff +169c | 01/29/25 11:42:15 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +169d | 01/29/25 11:42:15 | INF | BMC | Fan | FANS_SWB_PWM | - +169e | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +169f | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +16a0 | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 93 97 94 97 ff ff +16a1 | 01/29/25 11:42:15 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=42.00 C, threshold=110.00 C +16a2 | 01/29/25 11:42:15 | INF | BMC | Fan | FANS_SWB_PWM | - +16a3 | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +16a4 | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16a5 | 01/29/25 11:42:15 | INF | d0 | 000020 | | OEM Event 93 97 94 97 ff ff +16a6 | 01/29/25 11:42:15 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=42.00 C, threshold=107.00 C +16a7 | 01/29/25 11:42:15 | INF | BMC | Fan | FANS_SWB_PWM | - +16a8 | 01/29/25 11:42:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +16a9 | 01/29/25 11:42:16 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16aa | 01/29/25 11:42:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16ab | 01/29/25 11:42:16 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=42.00 C, threshold=105.00 C +16ac | 01/29/25 11:42:16 | INF | BMC | Fan | FANS_SWB_PWM | - +16ad | 01/29/25 11:42:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +16ae | 01/29/25 11:42:16 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16af | 01/29/25 11:42:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16b0 | 01/29/25 13:09:43 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +16b1 | 01/29/25 13:09:43 | INF | BMC | Fan | FANS_SWB_PWM | - +16b2 | 01/29/25 13:09:43 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16b3 | 01/29/25 13:09:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16b4 | 01/29/25 13:09:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16b5 | 01/29/25 13:09:44 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=34.00 C, threshold=5.00 C +16b6 | 01/29/25 13:09:44 | INF | BMC | Fan | FANS_SWB_PWM | - +16b7 | 01/29/25 13:09:44 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +16b8 | 01/29/25 13:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16b9 | 01/29/25 13:09:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16ba | 01/30/25 00:29:56 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +16bb | 01/30/25 00:29:56 | INF | BMC | Fan | FANS_SWB_PWM | - +16bc | 01/30/25 00:29:56 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +16bd | 01/30/25 00:29:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16be | 01/30/25 00:29:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16bf | 01/30/25 00:29:57 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=34.00 C, threshold=5.00 C +16c0 | 01/30/25 00:29:57 | INF | BMC | Fan | FANS_SWB_PWM | - +16c1 | 01/30/25 00:29:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +16c2 | 01/30/25 00:29:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16c3 | 01/30/25 00:29:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16c4 | 01/30/25 08:19:43 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +16c5 | 01/30/25 08:19:43 | INF | BMC | Fan | FANS_SWB_PWM | - +16c6 | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16c7 | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16c8 | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 96 94 96 ff ff +16c9 | 01/30/25 08:19:44 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +16ca | 01/30/25 08:19:44 | INF | BMC | Fan | FANS_SWB_PWM | - +16cb | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16cc | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16cd | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +16ce | 01/30/25 08:19:44 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +16cf | 01/30/25 08:19:44 | INF | BMC | Fan | FANS_SWB_PWM | - +16d0 | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16d1 | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16d2 | 01/30/25 08:19:44 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +16d3 | 01/30/25 08:19:44 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=42.00 C, threshold=110.00 C +16d4 | 01/30/25 08:19:44 | INF | BMC | Fan | FANS_SWB_PWM | - +16d5 | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16d6 | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16d7 | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +16d8 | 01/30/25 08:19:45 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=42.00 C, threshold=107.00 C +16d9 | 01/30/25 08:19:45 | INF | BMC | Fan | FANS_SWB_PWM | - +16da | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16db | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16dc | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +16dd | 01/30/25 08:19:45 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=42.00 C, threshold=105.00 C +16de | 01/30/25 08:19:45 | INF | BMC | Fan | FANS_SWB_PWM | - +16df | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +16e0 | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +16e1 | 01/30/25 08:19:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +16e2 | 01/30/25 21:06:55 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +16e3 | 01/30/25 21:06:55 | INF | BMC | Fan | FANS_SWB_PWM | - +16e4 | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16e5 | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16e6 | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16e7 | 01/30/25 21:06:56 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +16e8 | 01/30/25 21:06:56 | INF | BMC | Fan | FANS_SWB_PWM | - +16e9 | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16ea | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16eb | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16ec | 01/30/25 21:06:56 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +16ed | 01/30/25 21:06:56 | INF | BMC | Fan | FANS_SWB_PWM | - +16ee | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +16ef | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16f0 | 01/30/25 21:06:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16f1 | 01/30/25 21:06:56 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=31.00 C, threshold=110.00 C +16f2 | 01/30/25 21:06:56 | INF | BMC | Fan | FANS_SWB_PWM | - +16f3 | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +16f4 | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16f5 | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16f6 | 01/30/25 21:06:57 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=31.00 C, threshold=107.00 C +16f7 | 01/30/25 21:06:57 | INF | BMC | Fan | FANS_SWB_PWM | - +16f8 | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +16f9 | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16fa | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +16fb | 01/30/25 21:06:57 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=31.00 C, threshold=105.00 C +16fc | 01/30/25 21:06:57 | INF | BMC | Fan | FANS_SWB_PWM | - +16fd | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +16fe | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +16ff | 01/30/25 21:06:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1700 | 01/30/25 21:37:32 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1701 | 01/30/25 21:37:32 | INF | BMC | Fan | FANS_SWB_PWM | - +1702 | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1703 | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1704 | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1705 | 01/30/25 21:37:33 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1706 | 01/30/25 21:37:33 | INF | BMC | Fan | FANS_SWB_PWM | - +1707 | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1708 | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1709 | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +170a | 01/30/25 21:37:33 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +170b | 01/30/25 21:37:33 | INF | BMC | Fan | FANS_SWB_PWM | - +170c | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +170d | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +170e | 01/30/25 21:37:33 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +170f | 01/30/25 21:37:33 | INF | BMC | OEM(e0) | BMC_Health | - +1710 | 01/30/25 21:37:33 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +1711 | 01/30/25 21:37:33 | INF | BMC | Fan | FANS_SWB_PWM | - +1712 | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1713 | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1714 | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1715 | 01/30/25 21:37:34 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +1716 | 01/30/25 21:37:34 | INF | BMC | Fan | FANS_SWB_PWM | - +1717 | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1718 | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1719 | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +171a | 01/30/25 21:37:34 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +171b | 01/30/25 21:37:34 | INF | BMC | Fan | FANS_SWB_PWM | - +171c | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +171d | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +171e | 01/30/25 21:37:34 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +171f | 01/30/25 23:25:59 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1720 | 01/30/25 23:25:59 | INF | BMC | Fan | FANS_SWB_PWM | - +1721 | 01/30/25 23:25:59 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1722 | 01/30/25 23:25:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1723 | 01/30/25 23:25:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1724 | 01/30/25 23:25:59 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1725 | 01/30/25 23:25:59 | INF | BMC | Fan | FANS_SWB_PWM | - +1726 | 01/30/25 23:25:59 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1727 | 01/30/25 23:25:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1728 | 01/30/25 23:25:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1729 | 01/30/25 23:25:59 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +172a | 01/30/25 23:25:59 | INF | BMC | Fan | FANS_SWB_PWM | - +172b | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +172c | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +172d | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +172e | 01/30/25 23:26:00 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=31.00 C, threshold=110.00 C +172f | 01/30/25 23:26:00 | INF | BMC | Fan | FANS_SWB_PWM | - +1730 | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1731 | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1732 | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1733 | 01/30/25 23:26:00 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=31.00 C, threshold=107.00 C +1734 | 01/30/25 23:26:00 | INF | BMC | Fan | FANS_SWB_PWM | - +1735 | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1736 | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1737 | 01/30/25 23:26:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1738 | 01/30/25 23:26:00 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=31.00 C, threshold=105.00 C +1739 | 01/30/25 23:26:00 | INF | BMC | Fan | FANS_SWB_PWM | - +173a | 01/30/25 23:26:01 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +173b | 01/30/25 23:26:01 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +173c | 01/30/25 23:26:01 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +173d | 01/31/25 04:49:43 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +173e | 01/31/25 04:49:43 | INF | BMC | Fan | FANS_SWB_PWM | - +173f | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1740 | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1741 | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1742 | 01/31/25 04:49:43 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1743 | 01/31/25 04:49:43 | INF | BMC | Fan | FANS_SWB_PWM | - +1744 | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 98 +1745 | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1746 | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1747 | 01/31/25 04:49:43 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1748 | 01/31/25 04:49:43 | INF | BMC | Fan | FANS_SWB_PWM | - +1749 | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 98 +174a | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +174b | 01/31/25 04:49:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +174c | 01/31/25 04:49:44 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=35.00 C, threshold=110.00 C +174d | 01/31/25 04:49:44 | INF | BMC | Fan | FANS_SWB_PWM | - +174e | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +174f | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1750 | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1751 | 01/31/25 04:49:44 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=35.00 C, threshold=107.00 C +1752 | 01/31/25 04:49:44 | INF | BMC | Fan | FANS_SWB_PWM | - +1753 | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1754 | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1755 | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1756 | 01/31/25 04:49:44 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=35.00 C, threshold=105.00 C +1757 | 01/31/25 04:49:44 | INF | BMC | Fan | FANS_SWB_PWM | - +1758 | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1759 | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +175a | 01/31/25 04:49:44 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +175b | 01/31/25 11:22:40 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +175c | 01/31/25 11:22:40 | INF | BMC | Fan | FANS_SWB_PWM | - +175d | 01/31/25 11:22:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +175e | 01/31/25 11:22:40 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +175f | 01/31/25 11:22:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1760 | 01/31/25 11:22:41 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=34.00 C, threshold=5.00 C +1761 | 01/31/25 11:22:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1762 | 01/31/25 11:22:41 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1763 | 01/31/25 11:22:41 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +1764 | 01/31/25 11:22:41 | INF | d0 | 000020 | | OEM Event 93 96 94 96 ff ff +1765 | 01/31/25 12:21:13 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1766 | 01/31/25 12:21:13 | INF | BMC | Fan | FANS_SWB_PWM | - +1767 | 01/31/25 12:21:13 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1768 | 01/31/25 12:21:13 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1769 | 01/31/25 12:21:13 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +176a | 01/31/25 12:21:13 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +176b | 01/31/25 12:21:13 | INF | BMC | Fan | FANS_SWB_PWM | - +176c | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +176d | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +176e | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +176f | 01/31/25 12:21:14 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1770 | 01/31/25 12:21:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1771 | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1772 | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1773 | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1774 | 01/31/25 12:21:14 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=43.00 C, threshold=110.00 C +1775 | 01/31/25 12:21:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1776 | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1777 | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1778 | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1779 | 01/31/25 12:21:14 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=43.00 C, threshold=107.00 C +177a | 01/31/25 12:21:14 | INF | BMC | Fan | FANS_SWB_PWM | - +177b | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +177c | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +177d | 01/31/25 12:21:14 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +177e | 01/31/25 12:21:14 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=43.00 C, threshold=105.00 C +177f | 01/31/25 12:21:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1780 | 01/31/25 12:21:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1781 | 01/31/25 12:21:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1782 | 01/31/25 12:21:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1783 | 01/31/25 12:21:15 | INF | BMC | OEM(e0) | BMC_Health | - +1784 | 01/31/25 17:22:26 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1785 | 01/31/25 17:22:26 | INF | BMC | Fan | FANS_SWB_PWM | - +1786 | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1787 | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1788 | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1789 | 01/31/25 17:22:26 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +178a | 01/31/25 17:22:26 | INF | BMC | Fan | FANS_SWB_PWM | - +178b | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +178c | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +178d | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +178e | 01/31/25 17:22:26 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +178f | 01/31/25 17:22:26 | INF | BMC | Fan | FANS_SWB_PWM | - +1790 | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1791 | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1792 | 01/31/25 17:22:26 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1793 | 01/31/25 17:22:27 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1794 | 01/31/25 17:22:27 | INF | BMC | Fan | FANS_SWB_PWM | - +1795 | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1796 | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1797 | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1798 | 01/31/25 17:22:27 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1799 | 01/31/25 17:22:27 | INF | BMC | Fan | FANS_SWB_PWM | - +179a | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +179b | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +179c | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +179d | 01/31/25 17:22:27 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +179e | 01/31/25 17:22:27 | INF | BMC | Fan | FANS_SWB_PWM | - +179f | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +17a0 | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +17a1 | 01/31/25 17:22:27 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17a2 | 01/31/25 20:43:09 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +17a3 | 01/31/25 20:43:09 | INF | BMC | Fan | FANS_SWB_PWM | - +17a4 | 01/31/25 20:43:09 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +17a5 | 01/31/25 20:43:09 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +17a6 | 01/31/25 20:43:09 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +17a7 | 01/31/25 20:43:10 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=34.00 C, threshold=5.00 C +17a8 | 01/31/25 20:43:10 | INF | BMC | Fan | FANS_SWB_PWM | - +17a9 | 01/31/25 20:43:10 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +17aa | 01/31/25 20:43:10 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +17ab | 01/31/25 20:43:10 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17ac | 02/01/25 08:32:22 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +17ad | 02/01/25 08:32:22 | INF | BMC | Fan | FANS_SWB_PWM | - +17ae | 02/01/25 08:32:22 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +17af | 02/01/25 08:32:22 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +17b0 | 02/01/25 08:32:22 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17b1 | 02/01/25 08:32:23 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=34.00 C, threshold=5.00 C +17b2 | 02/01/25 08:32:23 | INF | BMC | Fan | FANS_SWB_PWM | - +17b3 | 02/01/25 08:32:23 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +17b4 | 02/01/25 08:32:23 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +17b5 | 02/01/25 08:32:23 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17b6 | 02/01/25 09:41:39 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +17b7 | 02/01/25 09:41:39 | INF | BMC | Fan | FANS_SWB_PWM | - +17b8 | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +17b9 | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +17ba | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17bb | 02/01/25 09:41:39 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +17bc | 02/01/25 09:41:39 | INF | BMC | Fan | FANS_SWB_PWM | - +17bd | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +17be | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +17bf | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17c0 | 02/01/25 09:41:39 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +17c1 | 02/01/25 09:41:39 | INF | BMC | Fan | FANS_SWB_PWM | - +17c2 | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +17c3 | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +17c4 | 02/01/25 09:41:39 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17c5 | 02/01/25 09:41:40 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=34.00 C, threshold=85.00 C +17c6 | 02/01/25 09:41:40 | INF | BMC | Fan | FANS_SWB_PWM | - +17c7 | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +17c8 | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +17c9 | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17ca | 02/01/25 09:41:40 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=34.00 C, threshold=80.00 C +17cb | 02/01/25 09:41:40 | INF | BMC | Fan | FANS_SWB_PWM | - +17cc | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +17cd | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +17ce | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17cf | 02/01/25 09:41:40 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=34.00 C, threshold=75.00 C +17d0 | 02/01/25 09:41:40 | INF | BMC | Fan | FANS_SWB_PWM | - +17d1 | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +17d2 | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +17d3 | 02/01/25 09:41:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +17d4 | 02/01/25 15:46:31 | INF | BMC | ACPI Power State | Power_Status | S5/G2 soft-off +17d5 | 02/01/25 15:46:31 | INF | BMC | OEM(df) | GP_PowerGood | - +17d6 | 02/01/25 15:46:31 | INF | BMC | OEM(df) | E1S_PowerGood | - +17d7 | 02/01/25 15:46:31 | INF | BMC | OEM(df) | E1S_PowerGood | - +17d8 | 02/01/25 15:46:31 | INF | BMC | OEM(df) | E1S_PowerGood | - +17d9 | 02/01/25 15:46:31 | INF | BMC | OEM(df) | E1S_PowerGood | - +17da | 02/01/25 15:46:31 | INF | BMC | OEM(df) | E1S_PowerGood | - +17db | 02/01/25 15:48:34 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +17dc | 02/01/25 15:48:35 | INF | BMC | OEM(df) | GP_PowerGood | - +17dd | 02/01/25 15:48:35 | INF | BMC | OEM(df) | E1S_PowerGood | - +17de | 02/01/25 15:48:35 | INF | BMC | OEM(df) | E1S_PowerGood | - +17df | 02/01/25 15:48:35 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e0 | 02/01/25 15:48:35 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e1 | 02/01/25 15:48:35 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e2 | 02/01/25 16:06:50 | INF | BMC | ACPI Power State | Power_Status | S5/G2 soft-off +17e3 | 02/01/25 16:06:50 | INF | BMC | OEM(df) | GP_PowerGood | - +17e4 | 02/01/25 16:06:50 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e5 | 02/01/25 16:06:50 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e6 | 02/01/25 16:06:50 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e7 | 02/01/25 16:06:50 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e8 | 02/01/25 16:06:50 | INF | BMC | OEM(df) | E1S_PowerGood | - +17e9 | 02/01/25 16:08:52 | INF | BMC | OEM(df) | GP_PowerGood | - +17ea | 02/01/25 16:08:52 | INF | BMC | OEM(df) | E1S_PowerGood | - +17eb | 02/01/25 16:08:52 | INF | BMC | OEM(df) | E1S_PowerGood | - +17ec | 02/01/25 16:08:52 | INF | BMC | OEM(df) | E1S_PowerGood | - +17ed | 02/01/25 16:08:52 | INF | BMC | OEM(df) | E1S_PowerGood | - +17ee | 02/01/25 16:08:52 | INF | BMC | OEM(df) | E1S_PowerGood | - +17ef | 02/01/25 16:08:53 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +17f0 | 02/01/25 16:49:51 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +17f1 | 02/01/25 16:54:08 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +17f2 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f3 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f4 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f5 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f6 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f7 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f8 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17f9 | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17fa | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17fb | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_PowerGood | - +17fc | 02/01/25 16:54:15 | INF | BMC | OEM(df) | E1S_Present | - +17fd | 02/01/25 16:54:20 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +17fe | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +17ff | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1800 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1801 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1802 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1803 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1804 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1805 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1806 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1807 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_PowerGood | - +1808 | 02/01/25 16:54:20 | INF | BMC | OEM(df) | E1S_Present | - +1809 | 02/01/25 16:54:21 | INF | BMC | OEM(e0) | BMC_Health | - +180a | 02/01/25 16:54:22 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +180b | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_3B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +180c | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_3A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +180d | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_6B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +180e | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_6A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +180f | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_7B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +1810 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_7A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +1811 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_2B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +1812 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_1A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +1813 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_2A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +1814 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_0B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +1815 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_0A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +1816 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_1B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +1817 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_5B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +1818 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_5A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +1819 | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_4B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +181a | 02/01/25 17:06:01 | MAJ | BMC | Fan | FAN_4A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +181b | 02/01/25 17:06:01 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +181c | 02/01/25 17:06:01 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +181d | 02/01/25 17:06:01 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +181e | 02/01/25 17:06:02 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +181f | 02/01/25 17:06:02 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1820 | 02/01/25 17:06:02 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1821 | 02/01/25 17:06:02 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1822 | 02/01/25 17:06:03 | INF | BMC | Fan | FANS_SWB_PWM | - +1823 | 02/01/25 17:06:03 | INF | d0 | 000020 | | OEM Event 00 00 00 00 00 00 +1824 | 02/01/25 17:06:03 | INF | d0 | 000020 | | OEM Event 00 00 00 00 00 00 +1825 | 02/01/25 17:06:03 | INF | d0 | 000020 | | OEM Event 00 00 00 00 ff ff +1826 | 02/01/25 17:06:05 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1827 | 02/01/25 17:06:05 | INF | BMC | OEM(df) | E1S_PowerGood | - +1828 | 02/01/25 17:06:05 | INF | BMC | OEM(df) | E1S_PowerGood | - +1829 | 02/01/25 17:06:05 | INF | BMC | OEM(df) | E1S_PowerGood | - +182a | 02/01/25 17:06:05 | INF | BMC | OEM(e0) | BMC_Health | - +182b | 02/01/25 17:06:05 | INF | BMC | OEM(df) | E1S_PowerGood | - +182c | 02/01/25 17:06:05 | INF | BMC | OEM(df) | E1S_PowerGood | - +182d | 02/01/25 17:06:06 | INF | BMC | OEM(df) | E1S_Present | - +182e | 02/01/25 17:06:06 | INF | BMC | OEM(df) | E1S_Present | - +182f | 02/01/25 17:06:06 | INF | BMC | OEM(e0) | BMC_Health | - +1830 | 02/01/25 17:06:06 | INF | BMC | OEM(df) | E1S_Present | - +1831 | 02/01/25 17:06:06 | INF | BMC | OEM(df) | E1S_Present | - +1832 | 02/01/25 17:06:06 | INF | BMC | OEM(df) | E1S_Present | - +1833 | 02/01/25 17:06:06 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1834 | 02/01/25 17:06:06 | MAJ | BMC | Power Supply | HSC_0_Status | Failure detected +1835 | 02/01/25 17:06:06 | INF | BMC | OEM(e0) | BMC_Health | - +1836 | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1837 | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1838 | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1839 | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +183a | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +183b | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +183c | 02/01/25 17:06:45 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +183d | 02/01/25 17:06:45 | INF | BMC | OEM(df) | E1S_Present | - +183e | 02/01/25 17:06:47 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +183f | 02/01/25 17:06:47 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1840 | 02/01/25 17:06:47 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +1841 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_3A | LoC thresh OK now actual=6660.00 RPM, threshold=4440.00 RPM +1842 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_2A | LoC thresh OK now actual=7400.00 RPM, threshold=4440.00 RPM +1843 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_1A | LoC thresh OK now actual=7400.00 RPM, threshold=4440.00 RPM +1844 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_1B | LoC thresh OK now actual=6885.00 RPM, threshold=4455.00 RPM +1845 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_2B | LoC thresh OK now actual=6750.00 RPM, threshold=4455.00 RPM +1846 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_0A | LoC thresh OK now actual=7548.00 RPM, threshold=4440.00 RPM +1847 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_0B | LoC thresh OK now actual=6885.00 RPM, threshold=4455.00 RPM +1848 | 02/01/25 17:06:47 | INF | BMC | Fan | FAN_3B | LoC thresh OK now actual=6075.00 RPM, threshold=4455.00 RPM +1849 | 02/01/25 17:06:49 | INF | BMC | Fan | FAN_5B | LoC thresh OK now actual=8640.00 RPM, threshold=4455.00 RPM +184a | 02/01/25 17:06:49 | INF | BMC | Fan | FAN_5A | LoC thresh OK now actual=9324.00 RPM, threshold=4440.00 RPM +184b | 02/01/25 17:06:49 | INF | BMC | Fan | FAN_4B | LoC thresh OK now actual=8640.00 RPM, threshold=4455.00 RPM +184c | 02/01/25 17:06:49 | INF | BMC | Fan | FAN_4A | LoC thresh OK now actual=9472.00 RPM, threshold=4440.00 RPM +184d | 02/01/25 17:06:49 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=0.00 C, threshold=5.00 C +184e | 02/01/25 17:06:49 | INF | BMC | Fan | FANS_SWB_PWM | - +184f | 02/01/25 17:06:50 | INF | d0 | 000020 | | OEM Event 8d 8d 51 8b 50 8a +1850 | 02/01/25 17:06:50 | INF | d0 | 000020 | | OEM Event 87 84 40 40 3f 40 +1851 | 02/01/25 17:06:50 | INF | d0 | 000020 | | OEM Event 00 00 00 00 ff ff +1852 | 02/01/25 17:06:50 | INF | BMC | Fan | FAN_6B | LoC thresh OK now actual=8775.00 RPM, threshold=4455.00 RPM +1853 | 02/01/25 17:06:50 | INF | BMC | Fan | FAN_6A | LoC thresh OK now actual=9472.00 RPM, threshold=4440.00 RPM +1854 | 02/01/25 17:06:50 | INF | BMC | Fan | FAN_7A | LoC thresh OK now actual=9472.00 RPM, threshold=4440.00 RPM +1855 | 02/01/25 17:06:50 | INF | BMC | Fan | FAN_7B | LoC thresh OK now actual=8775.00 RPM, threshold=4455.00 RPM +1856 | 02/01/25 17:06:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1857 | 02/01/25 17:06:50 | INF | d0 | 000020 | | OEM Event 8d 8d 8c 8b 8b 8a +1858 | 02/01/25 17:06:50 | INF | d0 | 000020 | | OEM Event 87 84 63 61 62 61 +1859 | 02/01/25 17:06:50 | INF | d0 | 000020 | | OEM Event 40 40 40 40 ff ff +185a | 02/01/25 17:06:51 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +185b | 02/01/25 17:06:51 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +185c | 02/01/25 17:06:51 | INF | BMC | ACPI Power State | Power_Status | S5/G2 soft-off +185d | 02/01/25 17:06:51 | INF | BMC | OEM(da) | GPUB_FPGA1_STATE | - +185e | 02/01/25 17:06:51 | INF | BMC | OEM(da) | GPUB_FPGA2_STATE | - +185f | 02/01/25 17:06:51 | INF | BMC | OEM(df) | E1S_PowerGood | - +1860 | 02/01/25 17:06:51 | INF | BMC | OEM(df) | E1S_PowerGood | - +1861 | 02/01/25 17:06:51 | INF | BMC | OEM(df) | E1S_PowerGood | - +1862 | 02/01/25 17:06:51 | INF | BMC | OEM(df) | E1S_PowerGood | - +1863 | 02/01/25 17:06:52 | INF | BMC | OEM(df) | E1S_PowerGood | - +1864 | 02/01/25 17:06:52 | INF | BMC | OEM(df) | GP_PowerGood | - +1865 | 02/01/25 17:06:52 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1866 | 02/01/25 17:06:55 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1867 | 02/01/25 17:06:58 | INF | BMC | System Event | System_Reconfig | - +1868 | 02/01/25 17:07:10 | INF | db | OEM Event    +1869 | 02/01/25 17:07:12 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +186a | 02/01/25 17:07:12 | INF | BMC | Fan | FANS_SWB_PWM | - +186b | 02/01/25 17:07:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +186c | 02/01/25 17:07:12 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +186d | 02/01/25 17:07:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +186e | 02/01/25 17:07:15 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +186f | 02/01/25 17:08:08 | INF | BMC | OEM(e2) | FailSafe_Status | - +1870 | 02/01/25 17:09:02 | INF | BMC | OEM(e0) | BMC_Health | - +1871 | 02/01/25 17:12:17 | INF | db | OEM Event   +1872 | 02/01/25 17:12:17 | INF | db | OEM Event  @ +1873 | 02/01/25 17:12:17 | INF | db | OEM Event  @  +1874 | 02/01/25 17:12:17 | INF | db | OEM Event  @  +1875 | 02/01/25 17:12:35 | INF | db | OEM Event    +1876 | 02/01/25 17:12:40 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1877 | 02/01/25 17:12:52 | INF | db | OEM Event   +1878 | 02/01/25 17:13:35 | INF | db | OEM Event   +1879 | 02/01/25 17:13:35 | INF | db | OEM Event   +187a | 02/01/25 17:13:41 | INF | db | OEM Event  @ +187b | 02/01/25 17:13:42 | INF | db | OEM Event   +187c | 02/01/25 17:13:42 | INF | db | OEM Event   +187d | 02/01/25 17:13:42 | INF | db | OEM Event  @ +187e | 02/01/25 17:13:42 | INF | db | OEM Event  @ +187f | 02/01/25 17:13:42 | INF | db | OEM Event  @ +1880 | 02/01/25 17:13:42 | INF | db | OEM Event  @ +1881 | 02/01/25 17:13:51 | INF | db | OEM Event  @ +1882 | 02/01/25 17:13:51 | INF | EFI | Boot Error | na | - +1883 | 02/01/25 17:13:51 | INF | db | OEM Event  @ +1884 | 02/01/25 17:13:51 | INF | db | OEM Event   +1885 | 02/01/25 17:13:51 | INF | BMC | OEM(e0) | BMC_Health | - +1886 | 02/01/25 17:13:51 | INF | db | OEM Event   +1887 | 02/01/25 17:13:51 | INF | db | OEM Event   +1888 | 02/01/25 17:13:54 | INF | db | OEM Event   +1889 | 02/01/25 18:02:11 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +188a | 02/01/25 18:02:11 | INF | BMC | Fan | FANS_SWB_PWM | - +188b | 02/01/25 18:02:11 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +188c | 02/01/25 18:02:11 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +188d | 02/01/25 18:02:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +188e | 02/01/25 18:02:12 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +188f | 02/01/25 18:02:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1890 | 02/01/25 18:02:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1891 | 02/01/25 18:02:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1892 | 02/01/25 18:02:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1893 | 02/02/25 06:52:30 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1894 | 02/02/25 06:52:30 | INF | BMC | Fan | FANS_SWB_PWM | - +1895 | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1896 | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1897 | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1898 | 02/02/25 06:52:30 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1899 | 02/02/25 06:52:30 | INF | BMC | Fan | FANS_SWB_PWM | - +189a | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +189b | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +189c | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +189d | 02/02/25 06:52:30 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +189e | 02/02/25 06:52:30 | INF | BMC | Fan | FANS_SWB_PWM | - +189f | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +18a0 | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +18a1 | 02/02/25 06:52:30 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18a2 | 02/02/25 06:52:31 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +18a3 | 02/02/25 06:52:31 | INF | BMC | Fan | FANS_SWB_PWM | - +18a4 | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +18a5 | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18a6 | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18a7 | 02/02/25 06:52:31 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +18a8 | 02/02/25 06:52:31 | INF | BMC | Fan | FANS_SWB_PWM | - +18a9 | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +18aa | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18ab | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18ac | 02/02/25 06:52:31 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +18ad | 02/02/25 06:52:31 | INF | BMC | Fan | FANS_SWB_PWM | - +18ae | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +18af | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18b0 | 02/02/25 06:52:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18b1 | 02/02/25 07:11:17 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +18b2 | 02/02/25 07:11:17 | INF | BMC | Fan | FANS_SWB_PWM | - +18b3 | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18b4 | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18b5 | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18b6 | 02/02/25 07:11:17 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +18b7 | 02/02/25 07:11:17 | INF | BMC | Fan | FANS_SWB_PWM | - +18b8 | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18b9 | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18ba | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18bb | 02/02/25 07:11:17 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +18bc | 02/02/25 07:11:17 | INF | BMC | Fan | FANS_SWB_PWM | - +18bd | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18be | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18bf | 02/02/25 07:11:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18c0 | 02/02/25 07:11:18 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=34.00 C, threshold=110.00 C +18c1 | 02/02/25 07:11:18 | INF | BMC | Fan | FANS_SWB_PWM | - +18c2 | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18c3 | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18c4 | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18c5 | 02/02/25 07:11:18 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=34.00 C, threshold=107.00 C +18c6 | 02/02/25 07:11:18 | INF | BMC | Fan | FANS_SWB_PWM | - +18c7 | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18c8 | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18c9 | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18ca | 02/02/25 07:11:18 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=34.00 C, threshold=105.00 C +18cb | 02/02/25 07:11:18 | INF | BMC | Fan | FANS_SWB_PWM | - +18cc | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18cd | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18ce | 02/02/25 07:11:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18cf | 02/02/25 19:17:01 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +18d0 | 02/02/25 19:17:01 | INF | BMC | Fan | FANS_SWB_PWM | - +18d1 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +18d2 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18d3 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18d4 | 02/02/25 19:17:02 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +18d5 | 02/02/25 19:17:02 | INF | BMC | Fan | FANS_SWB_PWM | - +18d6 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +18d7 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18d8 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18d9 | 02/02/25 19:17:02 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +18da | 02/02/25 19:17:02 | INF | BMC | Fan | FANS_SWB_PWM | - +18db | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +18dc | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18dd | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18de | 02/02/25 19:17:02 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +18df | 02/02/25 19:17:02 | INF | BMC | Fan | FANS_SWB_PWM | - +18e0 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +18e1 | 02/02/25 19:17:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18e2 | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18e3 | 02/02/25 19:17:03 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +18e4 | 02/02/25 19:17:03 | INF | BMC | Fan | FANS_SWB_PWM | - +18e5 | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18e6 | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18e7 | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18e8 | 02/02/25 19:17:03 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +18e9 | 02/02/25 19:17:03 | INF | BMC | Fan | FANS_SWB_PWM | - +18ea | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18eb | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +18ec | 02/02/25 19:17:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18ed | 02/02/25 19:35:49 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +18ee | 02/02/25 19:35:49 | INF | BMC | Fan | FANS_SWB_PWM | - +18ef | 02/02/25 19:35:49 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18f0 | 02/02/25 19:35:49 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +18f1 | 02/02/25 19:35:49 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18f2 | 02/02/25 19:35:49 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +18f3 | 02/02/25 19:35:49 | INF | BMC | Fan | FANS_SWB_PWM | - +18f4 | 02/02/25 19:35:49 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18f5 | 02/02/25 19:35:49 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +18f6 | 02/02/25 19:35:49 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18f7 | 02/02/25 19:35:49 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +18f8 | 02/02/25 19:35:49 | INF | BMC | Fan | FANS_SWB_PWM | - +18f9 | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +18fa | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +18fb | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +18fc | 02/02/25 19:35:50 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +18fd | 02/02/25 19:35:50 | INF | BMC | Fan | FANS_SWB_PWM | - +18fe | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 98 +18ff | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1900 | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1901 | 02/02/25 19:35:50 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +1902 | 02/02/25 19:35:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1903 | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1904 | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1905 | 02/02/25 19:35:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1906 | 02/02/25 19:35:50 | INF | BMC | OEM(e0) | BMC_Health | - +1907 | 02/02/25 19:35:50 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +1908 | 02/02/25 19:35:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1909 | 02/02/25 19:35:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +190a | 02/02/25 19:35:51 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +190b | 02/02/25 19:35:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +190c | 02/03/25 09:07:59 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +190d | 02/03/25 09:07:59 | INF | BMC | Fan | FANS_SWB_PWM | - +190e | 02/03/25 09:07:59 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +190f | 02/03/25 09:07:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1910 | 02/03/25 09:07:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1911 | 02/03/25 09:08:00 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +1912 | 02/03/25 09:08:00 | INF | BMC | Fan | FANS_SWB_PWM | - +1913 | 02/03/25 09:08:00 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1914 | 02/03/25 09:08:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1915 | 02/03/25 09:08:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1916 | 02/04/25 00:09:18 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1917 | 02/04/25 00:09:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1918 | 02/04/25 00:09:18 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1919 | 02/04/25 00:09:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +191a | 02/04/25 00:09:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +191b | 02/04/25 00:09:19 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +191c | 02/04/25 00:09:19 | INF | BMC | Fan | FANS_SWB_PWM | - +191d | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +191e | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +191f | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1920 | 02/04/25 00:09:19 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1921 | 02/04/25 00:09:19 | INF | BMC | Fan | FANS_SWB_PWM | - +1922 | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1923 | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1924 | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1925 | 02/04/25 00:09:19 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +1926 | 02/04/25 00:09:19 | INF | BMC | Fan | FANS_SWB_PWM | - +1927 | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1928 | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1929 | 02/04/25 00:09:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +192a | 02/04/25 00:09:19 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +192b | 02/04/25 00:09:19 | INF | BMC | Fan | FANS_SWB_PWM | - +192c | 02/04/25 00:09:20 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +192d | 02/04/25 00:09:20 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +192e | 02/04/25 00:09:20 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +192f | 02/04/25 00:09:20 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +1930 | 02/04/25 00:09:20 | INF | BMC | Fan | FANS_SWB_PWM | - +1931 | 02/04/25 00:09:20 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1932 | 02/04/25 00:09:20 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1933 | 02/04/25 00:09:20 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1934 | 02/04/25 00:35:04 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1935 | 02/04/25 00:35:04 | INF | BMC | Fan | FANS_SWB_PWM | - +1936 | 02/04/25 00:35:04 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1937 | 02/04/25 00:35:04 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1938 | 02/04/25 00:35:04 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1939 | 02/04/25 00:35:04 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +193a | 02/04/25 00:35:04 | INF | BMC | Fan | FANS_SWB_PWM | - +193b | 02/04/25 00:35:04 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +193c | 02/04/25 00:35:04 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +193d | 02/04/25 00:35:04 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +193e | 02/04/25 00:35:04 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +193f | 02/04/25 00:35:04 | INF | BMC | Fan | FANS_SWB_PWM | - +1940 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1941 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1942 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1943 | 02/04/25 00:35:05 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1944 | 02/04/25 00:35:05 | INF | BMC | Fan | FANS_SWB_PWM | - +1945 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1946 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1947 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1948 | 02/04/25 00:35:05 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1949 | 02/04/25 00:35:05 | INF | BMC | Fan | FANS_SWB_PWM | - +194a | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +194b | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +194c | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +194d | 02/04/25 00:35:05 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +194e | 02/04/25 00:35:05 | INF | BMC | Fan | FANS_SWB_PWM | - +194f | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1950 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1951 | 02/04/25 00:35:05 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1952 | 02/04/25 01:58:15 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +1953 | 02/04/25 01:58:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1954 | 02/04/25 01:58:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1955 | 02/04/25 01:58:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1956 | 02/04/25 01:58:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1957 | 02/04/25 01:58:16 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +1958 | 02/04/25 01:58:16 | INF | BMC | Fan | FANS_SWB_PWM | - +1959 | 02/04/25 01:58:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +195a | 02/04/25 01:58:16 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +195b | 02/04/25 01:58:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +195c | 02/04/25 02:32:06 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +195d | 02/04/25 02:32:06 | INF | BMC | Fan | FANS_SWB_PWM | - +195e | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +195f | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1960 | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1961 | 02/04/25 02:32:06 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1962 | 02/04/25 02:32:06 | INF | BMC | Fan | FANS_SWB_PWM | - +1963 | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1964 | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1965 | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1966 | 02/04/25 02:32:06 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1967 | 02/04/25 02:32:06 | INF | BMC | Fan | FANS_SWB_PWM | - +1968 | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1969 | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +196a | 02/04/25 02:32:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +196b | 02/04/25 02:32:07 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +196c | 02/04/25 02:32:07 | INF | BMC | Fan | FANS_SWB_PWM | - +196d | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +196e | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +196f | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1970 | 02/04/25 02:32:07 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +1971 | 02/04/25 02:32:07 | INF | BMC | Fan | FANS_SWB_PWM | - +1972 | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1973 | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1974 | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1975 | 02/04/25 02:32:07 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +1976 | 02/04/25 02:32:07 | INF | BMC | Fan | FANS_SWB_PWM | - +1977 | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1978 | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1979 | 02/04/25 02:32:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +197a | 02/04/25 13:30:46 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +197b | 02/04/25 13:30:46 | INF | BMC | Fan | FANS_SWB_PWM | - +197c | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +197d | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +197e | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +197f | 02/04/25 13:30:46 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1980 | 02/04/25 13:30:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1981 | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1982 | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1983 | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1984 | 02/04/25 13:30:46 | INF | BMC | OEM(e0) | BMC_Health | - +1985 | 02/04/25 13:30:46 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1986 | 02/04/25 13:30:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1987 | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1988 | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1989 | 02/04/25 13:30:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +198a | 02/04/25 13:30:47 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +198b | 02/04/25 13:30:47 | INF | BMC | Fan | FANS_SWB_PWM | - +198c | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +198d | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +198e | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +198f | 02/04/25 13:30:47 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1990 | 02/04/25 13:30:47 | INF | BMC | Fan | FANS_SWB_PWM | - +1991 | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1992 | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1993 | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1994 | 02/04/25 13:30:47 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1995 | 02/04/25 13:30:47 | INF | BMC | Fan | FANS_SWB_PWM | - +1996 | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1997 | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1998 | 02/04/25 13:30:47 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1999 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_3B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +199a | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_3A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +199b | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_6B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +199c | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_6A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +199d | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_7B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +199e | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_7A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +199f | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_2B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +19a0 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_1A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +19a1 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_2A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +19a2 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_0B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +19a3 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_0A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +19a4 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_1B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +19a5 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_5B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +19a6 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_5A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +19a7 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_4B | Lo Crit thresh actual=0.00 RPM, threshold=4455.00 RPM +19a8 | 02/04/25 15:11:40 | MAJ | BMC | Fan | FAN_4A | Lo Crit thresh actual=0.00 RPM, threshold=4440.00 RPM +19a9 | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19aa | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19ab | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19ac | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19ad | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19ae | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19af | 02/04/25 15:11:40 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19b0 | 02/04/25 15:11:41 | INF | BMC | Fan | FANS_SWB_PWM | - +19b1 | 02/04/25 15:11:41 | INF | d0 | 000020 | | OEM Event 00 00 00 00 00 00 +19b2 | 02/04/25 15:11:41 | INF | d0 | 000020 | | OEM Event 00 00 00 00 00 00 +19b3 | 02/04/25 15:11:41 | INF | d0 | 000020 | | OEM Event 00 00 00 00 ff ff +19b4 | 02/04/25 15:11:43 | INF | BMC | OEM(df) | E1S_PowerGood | - +19b5 | 02/04/25 15:11:43 | INF | BMC | OEM(e0) | BMC_Health | - +19b6 | 02/04/25 15:11:43 | INF | BMC | OEM(e0) | BMC_Health | - +19b7 | 02/04/25 15:11:43 | INF | BMC | OEM(df) | E1S_PowerGood | - +19b8 | 02/04/25 15:11:43 | INF | BMC | OEM(df) | E1S_PowerGood | - +19b9 | 02/04/25 15:11:43 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +19ba | 02/04/25 15:11:43 | INF | BMC | OEM(df) | E1S_PowerGood | - +19bb | 02/04/25 15:11:43 | INF | BMC | OEM(df) | E1S_PowerGood | - +19bc | 02/04/25 15:11:44 | INF | BMC | OEM(df) | E1S_Present | - +19bd | 02/04/25 15:11:44 | INF | BMC | OEM(df) | E1S_Present | - +19be | 02/04/25 15:11:44 | INF | BMC | OEM(df) | E1S_Present | - +19bf | 02/04/25 15:11:44 | INF | BMC | OEM(df) | E1S_Present | - +19c0 | 02/04/25 15:11:44 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +19c1 | 02/04/25 15:11:44 | INF | BMC | OEM(e0) | BMC_Health | - +19c2 | 02/04/25 15:11:44 | INF | BMC | OEM(df) | E1S_Present | - +19c3 | 02/04/25 15:11:44 | MAJ | BMC | Power Supply | HSC_0_Status | Failure detected +19c4 | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19c5 | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19c6 | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19c7 | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19c8 | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19c9 | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19ca | 02/04/25 15:12:26 | INF | BMC | OEM(df) | E1S_Present | - +19cb | 02/04/25 15:12:26 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19cc | 02/04/25 15:12:27 | INF | BMC | Fan | FAN_3B | LoC thresh OK now actual=7020.00 RPM, threshold=4455.00 RPM +19cd | 02/04/25 15:12:28 | INF | BMC | Fan | FAN_3A | LoC thresh OK now actual=7844.00 RPM, threshold=4440.00 RPM +19ce | 02/04/25 15:12:28 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19cf | 02/04/25 15:12:28 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19d0 | 02/04/25 15:12:28 | INF | BMC | FRU Sensor | SWB_CPLD_Status | - +19d1 | 02/04/25 15:12:29 | INF | BMC | Fan | FAN_1B | LoC thresh OK now actual=8505.00 RPM, threshold=4455.00 RPM +19d2 | 02/04/25 15:12:29 | INF | BMC | Fan | FAN_1A | LoC thresh OK now actual=13764.00 RPM, threshold=4440.00 RPM +19d3 | 02/04/25 15:12:29 | INF | BMC | Fan | FAN_2A | LoC thresh OK now actual=9176.00 RPM, threshold=4440.00 RPM +19d4 | 02/04/25 15:12:29 | INF | BMC | Fan | FAN_0B | LoC thresh OK now actual=8640.00 RPM, threshold=4455.00 RPM +19d5 | 02/04/25 15:12:29 | INF | BMC | Fan | FAN_0A | LoC thresh OK now actual=13912.00 RPM, threshold=4440.00 RPM +19d6 | 02/04/25 15:12:29 | INF | BMC | Fan | FAN_2B | LoC thresh OK now actual=8505.00 RPM, threshold=4455.00 RPM +19d7 | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_4A | LoC thresh OK now actual=9620.00 RPM, threshold=4440.00 RPM +19d8 | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_4B | LoC thresh OK now actual=8775.00 RPM, threshold=4455.00 RPM +19d9 | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_5A | LoC thresh OK now actual=9472.00 RPM, threshold=4440.00 RPM +19da | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_6B | LoC thresh OK now actual=8910.00 RPM, threshold=4455.00 RPM +19db | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_7B | LoC thresh OK now actual=8775.00 RPM, threshold=4455.00 RPM +19dc | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_5B | LoC thresh OK now actual=8775.00 RPM, threshold=4455.00 RPM +19dd | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_6A | LoC thresh OK now actual=9620.00 RPM, threshold=4440.00 RPM +19de | 02/04/25 15:12:30 | INF | BMC | Fan | FAN_7A | LoC thresh OK now actual=9620.00 RPM, threshold=4440.00 RPM +19df | 02/04/25 15:12:30 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=0.00 C, threshold=5.00 C +19e0 | 02/04/25 15:12:30 | INF | BMC | Fan | FANS_SWB_PWM | - +19e1 | 02/04/25 15:12:30 | INF | d0 | 000020 | | OEM Event 9e 9f 9c 9c 9b 9b +19e2 | 02/04/25 15:12:30 | INF | d0 | 000020 | | OEM Event 8f 8d 40 40 40 40 +19e3 | 02/04/25 15:12:30 | INF | d0 | 000020 | | OEM Event 40 41 41 41 ff ff +19e4 | 02/04/25 15:12:30 | INF | BMC | Fan | FANS_SWB_PWM | - +19e5 | 02/04/25 15:12:31 | INF | d0 | 000020 | | OEM Event 9e 9f 9c db da dd +19e6 | 02/04/25 15:12:31 | INF | d0 | 000020 | | OEM Event 8f ce 65 63 63 63 +19e7 | 02/04/25 15:12:31 | INF | d0 | 000020 | | OEM Event 66 66 67 66 ff ff +19e8 | 02/04/25 15:12:31 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +19e9 | 02/04/25 15:12:31 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +19ea | 02/04/25 15:12:31 | INF | BMC | ACPI Power State | Power_Status | S5/G2 soft-off +19eb | 02/04/25 15:12:31 | INF | BMC | OEM(da) | GPUB_FPGA1_STATE | - +19ec | 02/04/25 15:12:31 | INF | BMC | OEM(da) | GPUB_FPGA2_STATE | - +19ed | 02/04/25 15:12:32 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +19ee | 02/04/25 15:12:33 | INF | BMC | OEM(df) | E1S_PowerGood | - +19ef | 02/04/25 15:12:33 | INF | BMC | OEM(df) | E1S_PowerGood | - +19f0 | 02/04/25 15:12:33 | INF | BMC | OEM(df) | E1S_PowerGood | - +19f1 | 02/04/25 15:12:33 | INF | BMC | OEM(df) | E1S_PowerGood | - +19f2 | 02/04/25 15:12:33 | INF | BMC | OEM(df) | GP_PowerGood | - +19f3 | 02/04/25 15:12:33 | INF | BMC | OEM(df) | E1S_PowerGood | - +19f4 | 02/04/25 15:12:36 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +19f5 | 02/04/25 15:12:37 | INF | BMC | System Event | System_Reconfig | - +19f6 | 02/04/25 15:12:51 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +19f7 | 02/04/25 15:12:51 | INF | BMC | Fan | FANS_SWB_PWM | - +19f8 | 02/04/25 15:12:51 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +19f9 | 02/04/25 15:12:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +19fa | 02/04/25 15:12:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +19fb | 02/04/25 15:12:51 | INF | db | OEM Event    +19fc | 02/04/25 15:12:56 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +19fd | 02/04/25 15:13:48 | INF | BMC | OEM(e2) | FailSafe_Status | - +19fe | 02/04/25 15:14:42 | INF | BMC | OEM(e0) | BMC_Health | - +19ff | 02/04/25 15:17:57 | INF | db | OEM Event   +1a00 | 02/04/25 15:17:57 | INF | db | OEM Event  @ +1a01 | 02/04/25 15:17:57 | INF | db | OEM Event  @  +1a02 | 02/04/25 15:17:57 | INF | db | OEM Event  @  +1a03 | 02/04/25 15:17:58 | INF | BMC | OEM(e0) | BMC_Health | - +1a04 | 02/04/25 15:18:15 | INF | db | OEM Event    +1a05 | 02/04/25 15:18:20 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1a06 | 02/04/25 15:18:32 | INF | db | OEM Event   +1a07 | 02/04/25 15:19:09 | INF | c5 | 000137 | | OEM Event 03 00 00 00 00 00 +1a08 | 02/04/25 15:19:09 | INF | db | OEM Event   +1a09 | 02/04/25 15:19:11 | INF | BMC | OEM(e1) | BMC_Health | - +1a0a | 02/04/25 15:19:11 | INF | BMC | OEM(e1) | BMC_Health | - +1a0b | 02/04/25 15:19:11 | INF | BMC | OEM(e1) | BMC_Health | - +1a0c | 02/04/25 15:19:11 | INF | BMC | OEM(e1) | BMC_Health | - +1a0d | 02/04/25 15:19:14 | INF | db | OEM Event   +1a0e | 02/04/25 15:19:14 | INF | db | OEM Event   +1a0f | 02/04/25 15:19:14 | INF | db | OEM Event   +1a10 | 02/04/25 15:19:17 | INF | db | OEM Event   +1a11 | 02/04/25 23:55:28 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1a12 | 02/04/25 23:55:28 | INF | BMC | Fan | FANS_SWB_PWM | - +1a13 | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1a14 | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +1a15 | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a16 | 02/04/25 23:55:28 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1a17 | 02/04/25 23:55:28 | INF | BMC | Fan | FANS_SWB_PWM | - +1a18 | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1a19 | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +1a1a | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a1b | 02/04/25 23:55:28 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1a1c | 02/04/25 23:55:28 | INF | BMC | Fan | FANS_SWB_PWM | - +1a1d | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a1e | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 96 94 96 94 97 +1a1f | 02/04/25 23:55:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a20 | 02/04/25 23:55:29 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=44.00 C, threshold=110.00 C +1a21 | 02/04/25 23:55:29 | INF | BMC | Fan | FANS_SWB_PWM | - +1a22 | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a23 | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a24 | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a25 | 02/04/25 23:55:29 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=44.00 C, threshold=107.00 C +1a26 | 02/04/25 23:55:29 | INF | BMC | Fan | FANS_SWB_PWM | - +1a27 | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a28 | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a29 | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a2a | 02/04/25 23:55:29 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=44.00 C, threshold=105.00 C +1a2b | 02/04/25 23:55:29 | INF | BMC | Fan | FANS_SWB_PWM | - +1a2c | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1a2d | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a2e | 02/04/25 23:55:29 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a2f | 02/05/25 00:59:52 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1a30 | 02/05/25 00:59:52 | INF | BMC | Fan | FANS_SWB_PWM | - +1a31 | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1a32 | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a33 | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1a34 | 02/05/25 00:59:53 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1a35 | 02/05/25 00:59:53 | INF | BMC | Fan | FANS_SWB_PWM | - +1a36 | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1a37 | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a38 | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1a39 | 02/05/25 00:59:53 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1a3a | 02/05/25 00:59:53 | INF | BMC | Fan | FANS_SWB_PWM | - +1a3b | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1a3c | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a3d | 02/05/25 00:59:53 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1a3e | 02/05/25 00:59:53 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=43.00 C, threshold=110.00 C +1a3f | 02/05/25 00:59:53 | INF | BMC | Fan | FANS_SWB_PWM | - +1a40 | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a41 | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a42 | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a43 | 02/05/25 00:59:54 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=43.00 C, threshold=107.00 C +1a44 | 02/05/25 00:59:54 | INF | BMC | Fan | FANS_SWB_PWM | - +1a45 | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a46 | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a47 | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a48 | 02/05/25 00:59:54 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=43.00 C, threshold=105.00 C +1a49 | 02/05/25 00:59:54 | INF | BMC | Fan | FANS_SWB_PWM | - +1a4a | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a4b | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a4c | 02/05/25 00:59:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a4d | 02/05/25 02:44:34 | INF | BMC | Temperature | SoC_0_SOCHOT | Temp Asserted +1a4e | 02/05/25 02:44:34 | INF | BMC | ACPI Power State | Power_Status | S5/G2 soft-off +1a4f | 02/05/25 02:44:36 | INF | BMC | OEM(df) | GP_PowerGood | - +1a50 | 02/05/25 02:44:36 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a51 | 02/05/25 02:44:36 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a52 | 02/05/25 02:44:36 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a53 | 02/05/25 02:44:36 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a54 | 02/05/25 02:44:36 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a55 | 02/05/25 02:44:40 | INF | BMC | OEM(df) | GP_PowerGood | - +1a56 | 02/05/25 02:44:40 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a57 | 02/05/25 02:44:40 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a58 | 02/05/25 02:44:40 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a59 | 02/05/25 02:44:40 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a5a | 02/05/25 02:44:40 | INF | BMC | OEM(df) | E1S_PowerGood | - +1a5b | 02/05/25 02:44:41 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1a5c | 02/05/25 02:45:00 | INF | db | OEM Event    +1a5d | 02/05/25 02:45:05 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1a5e | 02/05/25 02:50:06 | INF | db | OEM Event   +1a5f | 02/05/25 02:50:06 | INF | db | OEM Event  @ +1a60 | 02/05/25 02:50:06 | INF | db | OEM Event  @  +1a61 | 02/05/25 02:50:06 | INF | db | OEM Event  @  +1a62 | 02/05/25 02:50:24 | INF | db | OEM Event    +1a63 | 02/05/25 02:50:29 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1a64 | 02/05/25 02:50:41 | INF | db | OEM Event   +1a65 | 02/05/25 02:51:40 | INF | db | OEM Event  @ +1a66 | 02/05/25 02:51:40 | INF | EFI | Boot Error | na | - +1a67 | 02/05/25 02:51:40 | INF | db | OEM Event  @ +1a68 | 02/05/25 02:51:41 | INF | db | OEM Event   +1a69 | 02/05/25 02:51:41 | INF | db | OEM Event   +1a6a | 02/05/25 02:51:41 | INF | db | OEM Event   +1a6b | 02/05/25 02:51:44 | INF | db | OEM Event   +1a6c | 02/05/25 10:34:14 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1a6d | 02/05/25 10:34:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1a6e | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a6f | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1a70 | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 93 97 94 96 ff ff +1a71 | 02/05/25 10:34:15 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1a72 | 02/05/25 10:34:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1a73 | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a74 | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1a75 | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1a76 | 02/05/25 10:34:15 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1a77 | 02/05/25 10:34:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1a78 | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a79 | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1a7a | 02/05/25 10:34:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a7b | 02/05/25 10:34:15 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=42.00 C, threshold=110.00 C +1a7c | 02/05/25 10:34:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1a7d | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a7e | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1a7f | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a80 | 02/05/25 10:34:16 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=42.00 C, threshold=107.00 C +1a81 | 02/05/25 10:34:16 | INF | BMC | Fan | FANS_SWB_PWM | - +1a82 | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a83 | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1a84 | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a85 | 02/05/25 10:34:16 | INF | BMC | OEM(e0) | BMC_Health | - +1a86 | 02/05/25 10:34:16 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=42.00 C, threshold=105.00 C +1a87 | 02/05/25 10:34:16 | INF | BMC | Fan | FANS_SWB_PWM | - +1a88 | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1a89 | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1a8a | 02/05/25 10:34:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a8b | 02/06/25 04:09:05 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1a8c | 02/06/25 04:09:05 | INF | BMC | Fan | FANS_SWB_PWM | - +1a8d | 02/06/25 04:09:05 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1a8e | 02/06/25 04:09:05 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a8f | 02/06/25 04:09:05 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1a90 | 02/06/25 04:09:05 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1a91 | 02/06/25 04:09:05 | INF | BMC | Fan | FANS_SWB_PWM | - +1a92 | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1a93 | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a94 | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1a95 | 02/06/25 04:09:06 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1a96 | 02/06/25 04:09:06 | INF | BMC | Fan | FANS_SWB_PWM | - +1a97 | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1a98 | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a99 | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1a9a | 02/06/25 04:09:06 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=44.00 C, threshold=110.00 C +1a9b | 02/06/25 04:09:06 | INF | BMC | Fan | FANS_SWB_PWM | - +1a9c | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1a9d | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1a9e | 02/06/25 04:09:06 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1a9f | 02/06/25 04:09:06 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=44.00 C, threshold=107.00 C +1aa0 | 02/06/25 04:09:06 | INF | BMC | Fan | FANS_SWB_PWM | - +1aa1 | 02/06/25 04:09:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1aa2 | 02/06/25 04:09:07 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1aa3 | 02/06/25 04:09:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1aa4 | 02/06/25 04:09:07 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=44.00 C, threshold=105.00 C +1aa5 | 02/06/25 04:09:07 | INF | BMC | Fan | FANS_SWB_PWM | - +1aa6 | 02/06/25 04:09:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1aa7 | 02/06/25 04:09:07 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1aa8 | 02/06/25 04:09:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1aa9 | 02/06/25 13:33:17 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1aaa | 02/06/25 13:33:17 | INF | BMC | Fan | FANS_SWB_PWM | - +1aab | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1aac | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1aad | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1aae | 02/06/25 13:33:17 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1aaf | 02/06/25 13:33:17 | INF | BMC | Fan | FANS_SWB_PWM | - +1ab0 | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1ab1 | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1ab2 | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1ab3 | 02/06/25 13:33:17 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1ab4 | 02/06/25 13:33:17 | INF | BMC | Fan | FANS_SWB_PWM | - +1ab5 | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1ab6 | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1ab7 | 02/06/25 13:33:17 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1ab8 | 02/06/25 13:33:18 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=45.00 C, threshold=110.00 C +1ab9 | 02/06/25 13:33:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1aba | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1abb | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1abc | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1abd | 02/06/25 13:33:18 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=45.00 C, threshold=107.00 C +1abe | 02/06/25 13:33:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1abf | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1ac0 | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1ac1 | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1ac2 | 02/06/25 13:33:18 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=45.00 C, threshold=105.00 C +1ac3 | 02/06/25 13:33:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1ac4 | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1ac5 | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1ac6 | 02/06/25 13:33:18 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ac7 | 02/06/25 14:40:55 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1ac8 | 02/06/25 14:40:55 | INF | BMC | Fan | FANS_SWB_PWM | - +1ac9 | 02/06/25 14:40:55 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1aca | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1acb | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1acc | 02/06/25 14:40:56 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1acd | 02/06/25 14:40:56 | INF | BMC | Fan | FANS_SWB_PWM | - +1ace | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1acf | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1ad0 | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1ad1 | 02/06/25 14:40:56 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1ad2 | 02/06/25 14:40:56 | INF | BMC | Fan | FANS_SWB_PWM | - +1ad3 | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1ad4 | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1ad5 | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1ad6 | 02/06/25 14:40:56 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1ad7 | 02/06/25 14:40:56 | INF | BMC | Fan | FANS_SWB_PWM | - +1ad8 | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1ad9 | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1ada | 02/06/25 14:40:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1adb | 02/06/25 14:40:56 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1adc | 02/06/25 14:40:56 | INF | BMC | Fan | FANS_SWB_PWM | - +1add | 02/06/25 14:40:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1ade | 02/06/25 14:40:57 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1adf | 02/06/25 14:40:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ae0 | 02/06/25 14:40:57 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1ae1 | 02/06/25 14:40:57 | INF | BMC | Fan | FANS_SWB_PWM | - +1ae2 | 02/06/25 14:40:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1ae3 | 02/06/25 14:40:57 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1ae4 | 02/06/25 14:40:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ae5 | 02/06/25 16:49:14 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1ae6 | 02/06/25 16:49:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1ae7 | 02/06/25 16:49:14 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1ae8 | 02/06/25 16:49:14 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1ae9 | 02/06/25 16:49:14 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1aea | 02/06/25 16:49:14 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1aeb | 02/06/25 16:49:14 | INF | BMC | Fan | FANS_SWB_PWM | - +1aec | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1aed | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1aee | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1aef | 02/06/25 16:49:15 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1af0 | 02/06/25 16:49:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1af1 | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1af2 | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1af3 | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1af4 | 02/06/25 16:49:15 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=44.00 C, threshold=110.00 C +1af5 | 02/06/25 16:49:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1af6 | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1af7 | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1af8 | 02/06/25 16:49:15 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1af9 | 02/06/25 16:49:15 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=44.00 C, threshold=107.00 C +1afa | 02/06/25 16:49:15 | INF | BMC | Fan | FANS_SWB_PWM | - +1afb | 02/06/25 16:49:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1afc | 02/06/25 16:49:16 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1afd | 02/06/25 16:49:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1afe | 02/06/25 16:49:16 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=44.00 C, threshold=105.00 C +1aff | 02/06/25 16:49:16 | INF | BMC | Fan | FANS_SWB_PWM | - +1b00 | 02/06/25 16:49:16 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1b01 | 02/06/25 16:49:16 | INF | d0 | 000020 | | OEM Event 94 97 94 96 94 97 +1b02 | 02/06/25 16:49:16 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b03 | 02/06/25 16:49:16 | INF | BMC | OEM(e0) | BMC_Health | - +1b04 | 02/06/25 19:22:46 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1b05 | 02/06/25 19:22:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1b06 | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b07 | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b08 | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1b09 | 02/06/25 19:22:46 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1b0a | 02/06/25 19:22:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1b0b | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b0c | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b0d | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b0e | 02/06/25 19:22:46 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1b0f | 02/06/25 19:22:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1b10 | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b11 | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b12 | 02/06/25 19:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b13 | 02/06/25 19:22:47 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=44.00 C, threshold=110.00 C +1b14 | 02/06/25 19:22:47 | INF | BMC | Fan | FANS_SWB_PWM | - +1b15 | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1b16 | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1b17 | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b18 | 02/06/25 19:22:47 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=44.00 C, threshold=107.00 C +1b19 | 02/06/25 19:22:47 | INF | BMC | Fan | FANS_SWB_PWM | - +1b1a | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1b1b | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1b1c | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b1d | 02/06/25 19:22:47 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=44.00 C, threshold=105.00 C +1b1e | 02/06/25 19:22:47 | INF | BMC | Fan | FANS_SWB_PWM | - +1b1f | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1b20 | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1b21 | 02/06/25 19:22:47 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b22 | 02/06/25 21:40:11 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1b23 | 02/06/25 21:40:11 | INF | BMC | Fan | FANS_SWB_PWM | - +1b24 | 02/06/25 21:40:11 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1b25 | 02/06/25 21:40:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b26 | 02/06/25 21:40:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b27 | 02/06/25 21:40:11 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1b28 | 02/06/25 21:40:11 | INF | BMC | Fan | FANS_SWB_PWM | - +1b29 | 02/06/25 21:40:11 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1b2a | 02/06/25 21:40:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b2b | 02/06/25 21:40:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b2c | 02/06/25 21:40:11 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1b2d | 02/06/25 21:40:11 | INF | BMC | Fan | FANS_SWB_PWM | - +1b2e | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1b2f | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b30 | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b31 | 02/06/25 21:40:12 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +1b32 | 02/06/25 21:40:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1b33 | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b34 | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b35 | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b36 | 02/06/25 21:40:12 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +1b37 | 02/06/25 21:40:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1b38 | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b39 | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b3a | 02/06/25 21:40:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b3b | 02/06/25 21:40:12 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +1b3c | 02/06/25 21:40:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1b3d | 02/06/25 21:40:13 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b3e | 02/06/25 21:40:13 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b3f | 02/06/25 21:40:13 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b40 | 02/07/25 10:23:02 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1b41 | 02/07/25 10:23:02 | INF | BMC | Fan | FANS_SWB_PWM | - +1b42 | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b43 | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b44 | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b45 | 02/07/25 10:23:02 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1b46 | 02/07/25 10:23:02 | INF | BMC | Fan | FANS_SWB_PWM | - +1b47 | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b48 | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b49 | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b4a | 02/07/25 10:23:02 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1b4b | 02/07/25 10:23:02 | INF | BMC | Fan | FANS_SWB_PWM | - +1b4c | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b4d | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1b4e | 02/07/25 10:23:02 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b4f | 02/07/25 10:23:03 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1b50 | 02/07/25 10:23:03 | INF | BMC | Fan | FANS_SWB_PWM | - +1b51 | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b52 | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b53 | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b54 | 02/07/25 10:23:03 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1b55 | 02/07/25 10:23:03 | INF | BMC | Fan | FANS_SWB_PWM | - +1b56 | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1b57 | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b58 | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b59 | 02/07/25 10:23:03 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1b5a | 02/07/25 10:23:03 | INF | BMC | Fan | FANS_SWB_PWM | - +1b5b | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1b5c | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b5d | 02/07/25 10:23:03 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b5e | 02/07/25 12:40:59 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1b5f | 02/07/25 12:40:59 | INF | BMC | Fan | FANS_SWB_PWM | - +1b60 | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b61 | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b62 | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b63 | 02/07/25 12:40:59 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1b64 | 02/07/25 12:40:59 | INF | BMC | Fan | FANS_SWB_PWM | - +1b65 | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b66 | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 98 +1b67 | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b68 | 02/07/25 12:40:59 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1b69 | 02/07/25 12:40:59 | INF | BMC | Fan | FANS_SWB_PWM | - +1b6a | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b6b | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 98 +1b6c | 02/07/25 12:40:59 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b6d | 02/07/25 12:41:00 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1b6e | 02/07/25 12:41:00 | INF | BMC | Fan | FANS_SWB_PWM | - +1b6f | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b70 | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 98 +1b71 | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b72 | 02/07/25 12:41:00 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1b73 | 02/07/25 12:41:00 | INF | BMC | Fan | FANS_SWB_PWM | - +1b74 | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b75 | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b76 | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b77 | 02/07/25 12:41:00 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +1b78 | 02/07/25 12:41:00 | INF | BMC | Fan | FANS_SWB_PWM | - +1b79 | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b7a | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b7b | 02/07/25 12:41:00 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b7c | 02/08/25 10:42:06 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1b7d | 02/08/25 10:42:06 | INF | BMC | Fan | FANS_SWB_PWM | - +1b7e | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1b7f | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b80 | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b81 | 02/08/25 10:42:07 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1b82 | 02/08/25 10:42:07 | INF | BMC | Fan | FANS_SWB_PWM | - +1b83 | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1b84 | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b85 | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b86 | 02/08/25 10:42:07 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1b87 | 02/08/25 10:42:07 | INF | BMC | Fan | FANS_SWB_PWM | - +1b88 | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1b89 | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b8a | 02/08/25 10:42:07 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b8b | 02/08/25 10:42:07 | INF | BMC | OEM(e0) | BMC_Health | - +1b8c | 02/08/25 10:42:07 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1b8d | 02/08/25 10:42:07 | INF | BMC | Fan | FANS_SWB_PWM | - +1b8e | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1b8f | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b90 | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b91 | 02/08/25 10:42:08 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1b92 | 02/08/25 10:42:08 | INF | BMC | Fan | FANS_SWB_PWM | - +1b93 | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1b94 | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b95 | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b96 | 02/08/25 10:42:08 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +1b97 | 02/08/25 10:42:08 | INF | BMC | Fan | FANS_SWB_PWM | - +1b98 | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1b99 | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1b9a | 02/08/25 10:42:08 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1b9b | 02/08/25 12:14:27 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1b9c | 02/08/25 12:14:27 | INF | BMC | Fan | FANS_SWB_PWM | - +1b9d | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1b9e | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1b9f | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ba0 | 02/08/25 12:14:27 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1ba1 | 02/08/25 12:14:27 | INF | BMC | Fan | FANS_SWB_PWM | - +1ba2 | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1ba3 | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1ba4 | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ba5 | 02/08/25 12:14:27 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1ba6 | 02/08/25 12:14:27 | INF | BMC | Fan | FANS_SWB_PWM | - +1ba7 | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1ba8 | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1ba9 | 02/08/25 12:14:27 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1baa | 02/08/25 12:14:28 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1bab | 02/08/25 12:14:28 | INF | BMC | Fan | FANS_SWB_PWM | - +1bac | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1bad | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bae | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1baf | 02/08/25 12:14:28 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1bb0 | 02/08/25 12:14:28 | INF | BMC | Fan | FANS_SWB_PWM | - +1bb1 | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1bb2 | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bb3 | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bb4 | 02/08/25 12:14:28 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1bb5 | 02/08/25 12:14:28 | INF | BMC | Fan | FANS_SWB_PWM | - +1bb6 | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1bb7 | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bb8 | 02/08/25 12:14:28 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1bb9 | 02/08/25 14:22:45 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1bba | 02/08/25 14:22:45 | INF | BMC | Fan | FANS_SWB_PWM | - +1bbb | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1bbc | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bbd | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1bbe | 02/08/25 14:22:45 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1bbf | 02/08/25 14:22:45 | INF | BMC | Fan | FANS_SWB_PWM | - +1bc0 | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bc1 | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1bc2 | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1bc3 | 02/08/25 14:22:45 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1bc4 | 02/08/25 14:22:45 | INF | BMC | Fan | FANS_SWB_PWM | - +1bc5 | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bc6 | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1bc7 | 02/08/25 14:22:45 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bc8 | 02/08/25 14:22:46 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1bc9 | 02/08/25 14:22:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1bca | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bcb | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 98 +1bcc | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bcd | 02/08/25 14:22:46 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1bce | 02/08/25 14:22:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1bcf | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bd0 | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 98 +1bd1 | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bd2 | 02/08/25 14:22:46 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1bd3 | 02/08/25 14:22:46 | INF | BMC | Fan | FANS_SWB_PWM | - +1bd4 | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bd5 | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 98 +1bd6 | 02/08/25 14:22:46 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bd7 | 02/08/25 18:15:11 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1bd8 | 02/08/25 18:15:11 | INF | BMC | Fan | FANS_SWB_PWM | - +1bd9 | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1bda | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bdb | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bdc | 02/08/25 18:15:11 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1bdd | 02/08/25 18:15:11 | INF | BMC | Fan | FANS_SWB_PWM | - +1bde | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1bdf | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1be0 | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1be1 | 02/08/25 18:15:11 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1be2 | 02/08/25 18:15:11 | INF | BMC | Fan | FANS_SWB_PWM | - +1be3 | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1be4 | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1be5 | 02/08/25 18:15:11 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1be6 | 02/08/25 18:15:12 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +1be7 | 02/08/25 18:15:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1be8 | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1be9 | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1bea | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1beb | 02/08/25 18:15:12 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +1bec | 02/08/25 18:15:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1bed | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bee | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1bef | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bf0 | 02/08/25 18:15:12 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +1bf1 | 02/08/25 18:15:12 | INF | BMC | Fan | FANS_SWB_PWM | - +1bf2 | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bf3 | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1bf4 | 02/08/25 18:15:12 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bf5 | 02/08/25 19:28:42 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +1bf6 | 02/08/25 19:28:42 | INF | BMC | Fan | FANS_SWB_PWM | - +1bf7 | 02/08/25 19:28:42 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1bf8 | 02/08/25 19:28:42 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bf9 | 02/08/25 19:28:42 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bfa | 02/08/25 19:28:43 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +1bfb | 02/08/25 19:28:43 | INF | BMC | Fan | FANS_SWB_PWM | - +1bfc | 02/08/25 19:28:43 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1bfd | 02/08/25 19:28:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1bfe | 02/08/25 19:28:43 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1bff | 02/08/25 21:04:50 | CRT | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1c00 | 02/08/25 21:04:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1c01 | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c02 | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c03 | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c04 | 02/08/25 21:04:50 | MAJ | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1c05 | 02/08/25 21:04:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1c06 | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c07 | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c08 | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c09 | 02/08/25 21:04:50 | INF | BMC | OEM(e0) | BMC_Health | - +1c0a | 02/08/25 21:04:50 | MIN | BMC | Temperature | GPU3_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1c0b | 02/08/25 21:04:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1c0c | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c0d | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c0e | 02/08/25 21:04:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c0f | 02/08/25 21:04:51 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1c10 | 02/08/25 21:04:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1c11 | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1c12 | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c13 | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c14 | 02/08/25 21:04:51 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1c15 | 02/08/25 21:04:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1c16 | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1c17 | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c18 | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c19 | 02/08/25 21:04:51 | INF | BMC | Temperature | GPU3_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +1c1a | 02/08/25 21:04:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1c1b | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1c1c | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c1d | 02/08/25 21:04:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c1e | 02/08/25 21:42:57 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1c1f | 02/08/25 21:42:57 | INF | BMC | Fan | FANS_SWB_PWM | - +1c20 | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c21 | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c22 | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c23 | 02/08/25 21:42:57 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1c24 | 02/08/25 21:42:57 | INF | BMC | Fan | FANS_SWB_PWM | - +1c25 | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c26 | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c27 | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c28 | 02/08/25 21:42:57 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1c29 | 02/08/25 21:42:57 | INF | BMC | Fan | FANS_SWB_PWM | - +1c2a | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c2b | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c2c | 02/08/25 21:42:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c2d | 02/08/25 21:42:58 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=35.00 C, threshold=110.00 C +1c2e | 02/08/25 21:42:58 | INF | BMC | Fan | FANS_SWB_PWM | - +1c2f | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c30 | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c31 | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c32 | 02/08/25 21:42:58 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=35.00 C, threshold=107.00 C +1c33 | 02/08/25 21:42:58 | INF | BMC | Fan | FANS_SWB_PWM | - +1c34 | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c35 | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c36 | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c37 | 02/08/25 21:42:58 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=35.00 C, threshold=105.00 C +1c38 | 02/08/25 21:42:58 | INF | BMC | Fan | FANS_SWB_PWM | - +1c39 | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c3a | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c3b | 02/08/25 21:42:58 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c3c | 02/09/25 00:36:53 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1c3d | 02/09/25 00:36:53 | INF | BMC | Fan | FANS_SWB_PWM | - +1c3e | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1c3f | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c40 | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c41 | 02/09/25 00:36:54 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1c42 | 02/09/25 00:36:54 | INF | BMC | Fan | FANS_SWB_PWM | - +1c43 | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 98 +1c44 | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c45 | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c46 | 02/09/25 00:36:54 | MIN | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1c47 | 02/09/25 00:36:54 | INF | BMC | Fan | FANS_SWB_PWM | - +1c48 | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c49 | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c4a | 02/09/25 00:36:54 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c4b | 02/09/25 00:36:54 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiU thresh OK now actual=34.00 C, threshold=110.00 C +1c4c | 02/09/25 00:36:54 | INF | BMC | Fan | FANS_SWB_PWM | - +1c4d | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1c4e | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c4f | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c50 | 02/09/25 00:36:55 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiC thresh OK now actual=34.00 C, threshold=107.00 C +1c51 | 02/09/25 00:36:55 | INF | BMC | Fan | FANS_SWB_PWM | - +1c52 | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1c53 | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c54 | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c55 | 02/09/25 00:36:55 | INF | BMC | Temperature | GPU0_DRAM0_TMP0 | HiN thresh OK now actual=34.00 C, threshold=105.00 C +1c56 | 02/09/25 00:36:55 | INF | BMC | Fan | FANS_SWB_PWM | - +1c57 | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1c58 | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c59 | 02/09/25 00:36:55 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c5a | 02/09/25 07:30:48 | CRT | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1c5b | 02/09/25 07:30:48 | INF | BMC | Fan | FANS_SWB_PWM | - +1c5c | 02/09/25 07:30:48 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c5d | 02/09/25 07:30:48 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1c5e | 02/09/25 07:30:48 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1c5f | 02/09/25 07:30:48 | MAJ | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1c60 | 02/09/25 07:30:48 | INF | BMC | Fan | FANS_SWB_PWM | - +1c61 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c62 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1c63 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1c64 | 02/09/25 07:30:49 | MIN | BMC | Temperature | GPU1_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1c65 | 02/09/25 07:30:49 | INF | BMC | Fan | FANS_SWB_PWM | - +1c66 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c67 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 96 95 97 94 97 +1c68 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c69 | 02/09/25 07:30:49 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1c6a | 02/09/25 07:30:49 | INF | BMC | Fan | FANS_SWB_PWM | - +1c6b | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1c6c | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1c6d | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c6e | 02/09/25 07:30:49 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1c6f | 02/09/25 07:30:49 | INF | BMC | Fan | FANS_SWB_PWM | - +1c70 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1c71 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1c72 | 02/09/25 07:30:49 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c73 | 02/09/25 07:30:49 | INF | BMC | Temperature | GPU1_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +1c74 | 02/09/25 07:30:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1c75 | 02/09/25 07:30:50 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1c76 | 02/09/25 07:30:50 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1c77 | 02/09/25 07:30:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c78 | 02/09/25 10:45:40 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1c79 | 02/09/25 10:45:40 | INF | BMC | Fan | FANS_SWB_PWM | - +1c7a | 02/09/25 10:45:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c7b | 02/09/25 10:45:40 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c7c | 02/09/25 10:45:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c7d | 02/09/25 10:45:40 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1c7e | 02/09/25 10:45:40 | INF | BMC | Fan | FANS_SWB_PWM | - +1c7f | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c80 | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c81 | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c82 | 02/09/25 10:45:41 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1c83 | 02/09/25 10:45:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1c84 | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c85 | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c86 | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c87 | 02/09/25 10:45:41 | INF | BMC | OEM(e0) | BMC_Health | - +1c88 | 02/09/25 10:45:41 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1c89 | 02/09/25 10:45:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1c8a | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1c8b | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c8c | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c8d | 02/09/25 10:45:41 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1c8e | 02/09/25 10:45:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1c8f | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1c90 | 02/09/25 10:45:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c91 | 02/09/25 10:45:42 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c92 | 02/09/25 10:45:42 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +1c93 | 02/09/25 10:45:42 | INF | BMC | Fan | FANS_SWB_PWM | - +1c94 | 02/09/25 10:45:42 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1c95 | 02/09/25 10:45:42 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1c96 | 02/09/25 10:45:42 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c97 | 02/09/25 13:18:40 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1c98 | 02/09/25 13:18:40 | INF | BMC | Fan | FANS_SWB_PWM | - +1c99 | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1c9a | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1c9b | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1c9c | 02/09/25 13:18:40 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1c9d | 02/09/25 13:18:40 | INF | BMC | Fan | FANS_SWB_PWM | - +1c9e | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1c9f | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1ca0 | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ca1 | 02/09/25 13:18:40 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1ca2 | 02/09/25 13:18:40 | INF | BMC | Fan | FANS_SWB_PWM | - +1ca3 | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1ca4 | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1ca5 | 02/09/25 13:18:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1ca6 | 02/09/25 13:18:41 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=32.00 C, threshold=110.00 C +1ca7 | 02/09/25 13:18:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1ca8 | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 98 +1ca9 | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1caa | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1cab | 02/09/25 13:18:41 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=32.00 C, threshold=107.00 C +1cac | 02/09/25 13:18:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1cad | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 98 +1cae | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1caf | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1cb0 | 02/09/25 13:18:41 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=32.00 C, threshold=105.00 C +1cb1 | 02/09/25 13:18:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1cb2 | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 98 +1cb3 | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1cb4 | 02/09/25 13:18:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1cb5 | 02/09/25 17:03:52 | INF | db | OEM Event   +1cb6 | 02/09/25 17:03:52 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1cb7 | 02/09/25 17:03:54 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1cb8 | 02/09/25 17:03:58 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1cb9 | 02/09/25 17:04:10 | INF | db | OEM Event    +1cba | 02/09/25 17:04:15 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1cbb | 02/09/25 17:05:05 | INF | c5 | 000137 | | OEM Event 03 00 00 00 00 00 +1cbc | 02/09/25 17:05:05 | INF | db | OEM Event   +1cbd | 02/09/25 17:05:07 | INF | BMC | OEM(e1) | BMC_Health | - +1cbe | 02/09/25 17:05:07 | INF | BMC | OEM(e1) | BMC_Health | - +1cbf | 02/09/25 17:05:07 | INF | BMC | OEM(e1) | BMC_Health | - +1cc0 | 02/09/25 17:05:08 | INF | BMC | OEM(e1) | BMC_Health | - +1cc1 | 02/09/25 17:05:10 | INF | db | OEM Event   +1cc2 | 02/09/25 17:05:10 | INF | db | OEM Event   +1cc3 | 02/09/25 17:05:10 | INF | db | OEM Event   +1cc4 | 02/09/25 17:05:13 | INF | db | OEM Event   +1cc5 | 02/09/25 17:48:41 | INF | db | OEM Event   +1cc6 | 02/09/25 17:48:41 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1cc7 | 02/09/25 17:48:59 | INF | db | OEM Event    +1cc8 | 02/09/25 17:49:04 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1cc9 | 02/09/25 17:49:16 | INF | db | OEM Event   +1cca | 02/09/25 17:49:53 | INF | c5 | 000137 | | OEM Event 03 00 00 00 00 00 +1ccb | 02/09/25 17:49:53 | INF | db | OEM Event   +1ccc | 02/09/25 17:49:55 | INF | BMC | OEM(e1) | BMC_Health | - +1ccd | 02/09/25 17:49:55 | INF | BMC | OEM(e1) | BMC_Health | - +1cce | 02/09/25 17:49:55 | INF | BMC | OEM(e1) | BMC_Health | - +1ccf | 02/09/25 17:49:55 | INF | BMC | OEM(e1) | BMC_Health | - +1cd0 | 02/09/25 17:49:57 | INF | db | OEM Event   +1cd1 | 02/09/25 17:49:57 | INF | db | OEM Event   +1cd2 | 02/09/25 17:49:58 | INF | db | OEM Event   +1cd3 | 02/09/25 17:50:00 | INF | db | OEM Event   +1cd4 | 02/09/25 18:00:14 | INF | db | OEM Event   +1cd5 | 02/09/25 18:00:15 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1cd6 | 02/09/25 18:00:33 | INF | db | OEM Event    +1cd7 | 02/09/25 18:00:37 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1cd8 | 02/09/25 18:00:49 | INF | db | OEM Event   +1cd9 | 02/09/25 18:01:39 | INF | db | OEM Event   +1cda | 02/09/25 18:01:39 | INF | db | OEM Event   +1cdb | 02/09/25 18:01:39 | INF | db | OEM Event  @ +1cdc | 02/09/25 18:01:39 | INF | db | OEM Event  @ +1cdd | 02/09/25 18:01:39 | INF | db | OEM Event  @ +1cde | 02/09/25 18:01:39 | INF | db | OEM Event  @ +1cdf | 02/09/25 18:02:38 | INF | db | OEM Event  @ +1ce0 | 02/09/25 18:02:38 | INF | EFI | Boot Error | na | - +1ce1 | 02/09/25 18:02:39 | INF | db | OEM Event  @ +1ce2 | 02/09/25 18:02:39 | INF | db | OEM Event   +1ce3 | 02/09/25 18:02:39 | INF | db | OEM Event   +1ce4 | 02/09/25 18:02:39 | INF | db | OEM Event   +1ce5 | 02/09/25 18:02:42 | INF | db | OEM Event   +1ce6 | 02/09/25 19:53:35 | INF | db | OEM Event   +1ce7 | 02/09/25 19:53:36 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1ce8 | 02/09/25 19:53:53 | INF | db | OEM Event    +1ce9 | 02/09/25 19:53:58 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1cea | 02/09/25 19:54:48 | INF | c5 | 000137 | | OEM Event 03 00 00 00 00 00 +1ceb | 02/09/25 19:54:48 | INF | db | OEM Event   +1cec | 02/09/25 19:54:51 | INF | BMC | OEM(e1) | BMC_Health | - +1ced | 02/09/25 19:54:51 | INF | BMC | OEM(e1) | BMC_Health | - +1cee | 02/09/25 19:54:52 | INF | BMC | OEM(e1) | BMC_Health | - +1cef | 02/09/25 19:54:52 | INF | BMC | OEM(e1) | BMC_Health | - +1cf0 | 02/09/25 19:54:54 | INF | db | OEM Event   +1cf1 | 02/09/25 19:54:54 | INF | db | OEM Event   +1cf2 | 02/09/25 19:54:54 | INF | db | OEM Event   +1cf3 | 02/09/25 19:54:57 | INF | db | OEM Event   +1cf4 | 02/09/25 21:17:05 | INF | db | OEM Event   +1cf5 | 02/09/25 21:17:06 | INF | BMC | ACPI Power State | Power_Status | S0/G0 Working +1cf6 | 02/09/25 21:17:08 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1cf7 | 02/09/25 21:17:08 | INF | BMC | OEM(df) | GP_PowerGood | - +1cf8 | 02/09/25 21:17:12 | INF | BMC | OEM(da) | GPUB_HMC_STATE | - +1cf9 | 02/09/25 21:17:12 | INF | BMC | OEM(df) | GP_PowerGood | - +1cfa | 02/09/25 21:17:23 | INF | db | OEM Event    +1cfb | 02/09/25 21:17:28 | INF | ca | 000137 | | OEM Event 02 02 10 00 00 00 +1cfc | 02/09/25 21:17:40 | INF | db | OEM Event   +1cfd | 02/09/25 21:18:18 | INF | c5 | 000137 | | OEM Event 03 00 00 00 00 00 +1cfe | 02/09/25 21:18:18 | INF | db | OEM Event   +1cff | 02/09/25 21:18:19 | INF | BMC | OEM(e1) | BMC_Health | - +1d00 | 02/09/25 21:18:19 | INF | BMC | OEM(e1) | BMC_Health | - +1d01 | 02/09/25 21:18:19 | INF | BMC | OEM(e1) | BMC_Health | - +1d02 | 02/09/25 21:18:20 | INF | BMC | OEM(e1) | BMC_Health | - +1d03 | 02/09/25 21:18:20 | INF | BMC | OEM(e0) | BMC_Health | - +1d04 | 02/09/25 21:18:22 | INF | db | OEM Event   +1d05 | 02/09/25 21:18:22 | INF | db | OEM Event   +1d06 | 02/09/25 21:18:22 | INF | db | OEM Event   +1d07 | 02/09/25 21:18:25 | INF | db | OEM Event   +1d08 | 02/09/25 22:55:17 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1d09 | 02/09/25 22:55:17 | INF | BMC | Fan | FANS_SWB_PWM | - +1d0a | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d0b | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d0c | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1d0d | 02/09/25 22:55:18 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1d0e | 02/09/25 22:55:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1d0f | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d10 | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d11 | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1d12 | 02/09/25 22:55:18 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1d13 | 02/09/25 22:55:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1d14 | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d15 | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d16 | 02/09/25 22:55:18 | INF | d0 | 000020 | | OEM Event 94 96 94 97 ff ff +1d17 | 02/09/25 22:55:18 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1d18 | 02/09/25 22:55:18 | INF | BMC | Fan | FANS_SWB_PWM | - +1d19 | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1d1a | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d1b | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d1c | 02/09/25 22:55:19 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1d1d | 02/09/25 22:55:19 | INF | BMC | Fan | FANS_SWB_PWM | - +1d1e | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1d1f | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d20 | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d21 | 02/09/25 22:55:19 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1d22 | 02/09/25 22:55:19 | INF | BMC | Fan | FANS_SWB_PWM | - +1d23 | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1d24 | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d25 | 02/09/25 22:55:19 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d26 | 02/09/25 23:46:50 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1d27 | 02/09/25 23:46:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1d28 | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1d29 | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 98 +1d2a | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d2b | 02/09/25 23:46:50 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1d2c | 02/09/25 23:46:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1d2d | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1d2e | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 98 +1d2f | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d30 | 02/09/25 23:46:50 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1d31 | 02/09/25 23:46:50 | INF | BMC | Fan | FANS_SWB_PWM | - +1d32 | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1d33 | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d34 | 02/09/25 23:46:50 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d35 | 02/09/25 23:46:51 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1d36 | 02/09/25 23:46:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1d37 | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1d38 | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d39 | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d3a | 02/09/25 23:46:51 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1d3b | 02/09/25 23:46:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1d3c | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 94 97 +1d3d | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d3e | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d3f | 02/09/25 23:46:51 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1d40 | 02/09/25 23:46:51 | INF | BMC | Fan | FANS_SWB_PWM | - +1d41 | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1d42 | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d43 | 02/09/25 23:46:51 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d44 | 02/10/25 03:23:40 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +1d45 | 02/10/25 03:23:40 | INF | BMC | Fan | FANS_SWB_PWM | - +1d46 | 02/10/25 03:23:40 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d47 | 02/10/25 03:23:40 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d48 | 02/10/25 03:23:40 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d49 | 02/10/25 03:23:41 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +1d4a | 02/10/25 03:23:41 | INF | BMC | Fan | FANS_SWB_PWM | - +1d4b | 02/10/25 03:23:41 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1d4c | 02/10/25 03:23:41 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d4d | 02/10/25 03:23:41 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d4e | 02/10/25 04:03:56 | MIN | BMC | Temperature | HMC_TMP | Lo Noncrit thresh actual=128.00 C, threshold=5.00 C +1d4f | 02/10/25 04:03:56 | INF | BMC | Fan | FANS_SWB_PWM | - +1d50 | 02/10/25 04:03:56 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 97 +1d51 | 02/10/25 04:03:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d52 | 02/10/25 04:03:56 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d53 | 02/10/25 04:03:57 | INF | BMC | Temperature | HMC_TMP | LoN thresh OK now actual=31.00 C, threshold=5.00 C +1d54 | 02/10/25 04:03:57 | INF | BMC | Fan | FANS_SWB_PWM | - +1d55 | 02/10/25 04:03:57 | INF | d0 | 000020 | | OEM Event 94 98 94 97 93 97 +1d56 | 02/10/25 04:03:57 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d57 | 02/10/25 04:03:57 | INF | d0 | 000020 | | OEM Event 94 97 94 96 ff ff +1d58 | 02/10/25 11:27:22 | CRT | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Unrec thresh actual=127.00 C, threshold=110.00 C +1d59 | 02/10/25 11:27:22 | INF | BMC | Fan | FANS_SWB_PWM | - +1d5a | 02/10/25 11:27:22 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d5b | 02/10/25 11:27:22 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d5c | 02/10/25 11:27:22 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d5d | 02/10/25 11:27:22 | MAJ | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C +1d5e | 02/10/25 11:27:22 | INF | BMC | Fan | FANS_SWB_PWM | - +1d5f | 02/10/25 11:27:22 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d60 | 02/10/25 11:27:22 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d61 | 02/10/25 11:27:22 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d62 | 02/10/25 11:27:22 | MIN | BMC | Temperature | GPU2_DRAM0_TMP0 | Hi Noncrit thresh actual=127.00 C, threshold=105.00 C +1d63 | 02/10/25 11:27:22 | INF | BMC | Fan | FANS_SWB_PWM | - +1d64 | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d65 | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d66 | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d67 | 02/10/25 11:27:23 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiU thresh OK now actual=33.00 C, threshold=110.00 C +1d68 | 02/10/25 11:27:23 | INF | BMC | Fan | FANS_SWB_PWM | - +1d69 | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1d6a | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d6b | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d6c | 02/10/25 11:27:23 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiC thresh OK now actual=33.00 C, threshold=107.00 C +1d6d | 02/10/25 11:27:23 | INF | BMC | Fan | FANS_SWB_PWM | - +1d6e | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1d6f | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d70 | 02/10/25 11:27:23 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d71 | 02/10/25 11:27:23 | INF | BMC | Temperature | GPU2_DRAM0_TMP0 | HiN thresh OK now actual=33.00 C, threshold=105.00 C +1d72 | 02/10/25 11:27:23 | INF | BMC | Fan | FANS_SWB_PWM | - +1d73 | 02/10/25 11:27:24 | INF | d0 | 000020 | | OEM Event 94 99 94 97 94 98 +1d74 | 02/10/25 11:27:24 | INF | d0 | 000020 | | OEM Event 94 97 95 97 94 97 +1d75 | 02/10/25 11:27:24 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d76 | 02/10/25 18:02:30 | CRT | BMC | Temperature | HMC_TMP | Hi Unrec thresh actual=127.00 C, threshold=85.00 C +1d77 | 02/10/25 18:02:30 | INF | BMC | Fan | FANS_SWB_PWM | - +1d78 | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d79 | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1d7a | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d7b | 02/10/25 18:02:30 | MAJ | BMC | Temperature | HMC_TMP | Hi Crit thresh actual=127.00 C, threshold=80.00 C +1d7c | 02/10/25 18:02:30 | INF | BMC | Fan | FANS_SWB_PWM | - +1d7d | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d7e | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1d7f | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d80 | 02/10/25 18:02:30 | MIN | BMC | Temperature | HMC_TMP | Hi Noncrit thresh actual=127.00 C, threshold=75.00 C +1d81 | 02/10/25 18:02:30 | INF | BMC | Fan | FANS_SWB_PWM | - +1d82 | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d83 | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 96 94 97 94 97 +1d84 | 02/10/25 18:02:30 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d85 | 02/10/25 18:02:30 | INF | BMC | OEM(e0) | BMC_Health | - +1d86 | 02/10/25 18:02:31 | INF | BMC | Temperature | HMC_TMP | HiU thresh OK now actual=31.00 C, threshold=85.00 C +1d87 | 02/10/25 18:02:31 | INF | BMC | Fan | FANS_SWB_PWM | - +1d88 | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d89 | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d8a | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d8b | 02/10/25 18:02:31 | INF | BMC | Temperature | HMC_TMP | HiC thresh OK now actual=31.00 C, threshold=80.00 C +1d8c | 02/10/25 18:02:31 | INF | BMC | Fan | FANS_SWB_PWM | - +1d8d | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d8e | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d8f | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +1d90 | 02/10/25 18:02:31 | INF | BMC | Temperature | HMC_TMP | HiN thresh OK now actual=31.00 C, threshold=75.00 C +1d91 | 02/10/25 18:02:31 | INF | BMC | Fan | FANS_SWB_PWM | - +1d92 | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 99 94 97 93 97 +1d93 | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 94 97 +1d94 | 02/10/25 18:02:31 | INF | d0 | 000020 | | OEM Event 94 97 94 97 ff ff +ipmiutil sel, completed successfully diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_example_1.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_example_1.txt new file mode 100644 index 0000000000..e8ec9ffe28 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sel_example_1.txt @@ -0,0 +1,240 @@ +ipmiutil sel version 3.13 +-- BMC version 3.46, IPMI version 2.0 +SEL Ver fe Support 0f, Size = 2046 records (Used=230, Free=1816) +RecId Date/Time_______ SEV Src_ Evt_Type___ Sens# Evt_detail - Trig [Evt_data] +01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff +02 00 02 33 19 81 65 41 00 04 20 00 6f 03 ff ff +03 00 dd 33 19 81 65 37 01 00 00 01 00 02 80 00 +04 00 dd 33 19 81 65 37 01 00 01 59 00 6f 00 00 +05 00 dd 33 19 81 65 37 01 00 02 75 00 72 00 00 +06 00 dd 33 19 81 65 37 01 00 03 20 00 63 00 00 +07 00 dd 33 19 81 65 37 01 00 04 6f 00 6d 00 00 +08 00 dd 33 19 81 65 37 01 00 05 70 00 75 00 00 +09 00 dd 33 19 81 65 37 01 00 06 74 00 65 00 00 +0a 00 dd 33 19 81 65 37 01 00 07 72 00 20 00 00 +0b 00 dd 33 19 81 65 37 01 00 08 77 00 69 00 00 +0c 00 dd 33 19 81 65 37 01 00 09 6c 00 6c 00 00 +0d 00 dd 33 19 81 65 37 01 00 0a 20 00 72 00 00 +0e 00 dd 33 19 81 65 37 01 00 0b 65 00 73 00 00 +0f 00 dd 33 19 81 65 37 01 00 0c 74 00 61 00 00 +10 00 dd 34 19 81 65 37 01 00 0d 72 00 74 00 00 +11 00 dd 34 19 81 65 37 01 00 0e 20 00 61 00 00 +12 00 02 34 19 f8 64 20 00 04 07 c1 6f 00 ff ff +13 00 dd 34 19 81 65 37 01 00 10 31 00 32 00 00 +14 00 dd 34 19 81 65 37 01 00 11 2f 00 31 00 00 +15 00 dd 34 19 81 65 37 01 00 12 39 00 2f 00 00 +16 00 dd 34 19 81 65 37 01 00 13 32 00 30 00 00 +17 00 dd 34 19 81 65 37 01 00 14 32 00 33 00 00 +18 00 dd 34 19 81 65 37 01 00 15 20 00 34 00 00 +19 00 dd 34 19 81 65 37 01 00 16 3a 00 31 00 00 +1a 00 dd 34 19 81 65 37 01 00 17 34 00 3a 00 00 +1b 00 dd 34 19 81 65 37 01 00 18 35 00 33 00 00 +1c 00 dd 34 19 81 65 37 01 00 19 20 00 41 00 00 +1d 00 dd 34 19 81 65 37 01 00 1a 4d 00 20 00 00 +1e 00 dd 34 19 81 65 37 01 00 1b 74 00 6f 00 00 +1f 00 dd 34 19 81 65 37 01 00 1c 20 00 63 00 00 +20 00 dd 34 19 81 65 37 01 00 1d 6f 00 6d 00 00 +21 00 dd 34 19 81 65 37 01 00 1e 70 00 6c 00 00 +22 00 dd 34 19 81 65 37 01 00 1f 65 00 74 00 00 +23 00 dd 34 19 81 65 37 01 00 20 65 00 20 00 00 +24 00 dd 34 19 81 65 37 01 00 21 74 00 68 00 00 +25 00 dd 34 19 81 65 37 01 00 22 65 00 20 00 00 +26 00 dd 34 19 81 65 37 01 00 23 69 00 6e 00 00 +27 00 dd 34 19 81 65 37 01 00 24 73 00 74 00 00 +28 00 dd 34 19 81 65 37 01 00 25 61 00 6c 00 00 +29 00 dd 34 19 81 65 37 01 00 26 6c 00 61 00 00 +2a 00 dd 34 19 81 65 37 01 00 27 74 00 69 00 00 +2b 00 dd 34 19 81 65 37 01 00 28 6f 00 6e 00 00 +2c 00 dd 34 19 81 65 37 01 00 29 20 00 6f 00 00 +2d 00 dd 35 19 81 65 37 01 00 2a 66 00 20 00 00 +2e 00 dd 35 19 81 65 37 01 00 2b 61 00 70 00 00 +2f 00 dd 35 19 81 65 37 01 00 2c 70 00 6c 00 00 +30 00 dd 35 19 81 65 37 01 00 2d 69 00 63 00 00 +31 00 dd 35 19 81 65 37 01 00 2e 61 00 74 00 00 +32 00 dd 35 19 81 65 37 01 00 2f 69 00 6f 00 00 +33 00 dd 35 19 81 65 37 01 00 30 6e 00 73 00 00 +34 00 dd 35 19 81 65 37 01 00 31 20 00 61 00 00 +35 00 dd 35 19 81 65 37 01 00 32 6e 00 64 00 00 +36 00 dd 35 19 81 65 37 01 00 33 20 00 73 00 00 +37 00 dd 35 19 81 65 37 01 00 34 6f 00 66 00 00 +38 00 dd 35 19 81 65 37 01 00 35 74 00 77 00 00 +39 00 dd 35 19 81 65 37 01 00 36 61 00 72 00 00 +3a 00 dd 35 19 81 65 37 01 00 37 65 00 20 00 00 +3b 00 dd 35 19 81 65 37 01 00 38 75 00 70 00 00 +3c 00 dd 35 19 81 65 37 01 00 39 64 00 61 00 00 +3d 00 dd 35 19 81 65 37 01 00 3a 74 00 65 00 00 +3e 00 dd 35 19 81 65 37 01 00 3b 73 00 2e 00 00 +3f 00 c8 43 19 81 65 9c 9c 00 00 02 00 00 00 00 +40 00 02 43 19 81 65 20 00 04 28 ee ef c4 35 ff +41 00 02 4b 19 81 65 20 00 04 28 ee 6f c4 35 ff +42 00 02 64 19 81 65 20 00 04 e1 ef 6f 03 00 ff +43 00 02 64 19 81 65 20 00 04 e1 ef 6f 03 10 ff +44 00 02 64 19 81 65 20 00 04 e1 ef 6f 03 20 ff +45 00 02 64 19 81 65 20 00 04 e1 ef 6f 03 40 ff +46 00 02 a1 19 81 65 20 00 04 e1 ef 6f 03 00 ff +47 00 02 a1 19 81 65 20 00 04 e1 ef 6f 03 10 ff +48 00 02 a1 19 81 65 20 00 04 e1 ef 6f 03 20 ff +49 00 02 a1 19 81 65 20 00 04 e1 ef 6f 03 40 ff +4a 00 02 d2 19 81 65 41 00 04 1f 00 6f 01 ff ff +4b 00 dc d2 19 81 65 37 01 00 00 b0 19 81 65 00 +4c 00 02 58 b1 82 65 41 00 04 20 00 6f 03 ff ff +4d 00 dd 58 b1 82 65 37 01 00 00 01 00 04 84 00 +4e 00 02 5f b1 82 65 20 00 04 28 ee ef c4 35 ff +4f 00 c8 5f b1 82 65 9c 9c 00 00 02 00 00 00 00 +50 00 02 67 b1 82 65 20 00 04 28 ee 6f c4 35 ff +51 00 02 80 b1 82 65 20 00 04 e1 ef 6f 03 00 ff +52 00 02 80 b1 82 65 20 00 04 e1 ef 6f 03 10 ff +53 00 02 80 b1 82 65 20 00 04 e1 ef 6f 03 20 ff +54 00 02 80 b1 82 65 20 00 04 e1 ef 6f 03 40 ff +55 00 02 bd b1 82 65 20 00 04 e1 ef 6f 03 00 ff +56 00 02 bd b1 82 65 20 00 04 e1 ef 6f 03 10 ff +57 00 02 bd b1 82 65 20 00 04 e1 ef 6f 03 20 ff +58 00 02 be b1 82 65 20 00 04 e1 ef 6f 03 40 ff +59 00 02 ed b1 82 65 41 00 04 1f 00 6f 01 ff ff +5a 00 dc ed b1 82 65 37 01 00 00 cc b1 82 65 00 +5b 00 02 00 6b 96 65 20 00 04 28 ee 6f c4 af ff +5c 00 02 32 6b 96 65 20 00 04 28 ee ef c4 af ff +5d 00 02 de 15 a6 65 41 00 04 20 00 6f 03 ff ff +5e 00 dd de 15 a6 65 37 01 00 00 01 00 02 80 00 +5f 00 dd de 15 a6 65 37 01 00 01 59 00 6f 00 00 +60 00 dd de 15 a6 65 37 01 00 02 75 00 72 00 00 +61 00 dd de 15 a6 65 37 01 00 03 20 00 63 00 00 +62 00 dd de 15 a6 65 37 01 00 04 6f 00 6d 00 00 +63 00 dd de 15 a6 65 37 01 00 05 70 00 75 00 00 +64 00 dd de 15 a6 65 37 01 00 06 74 00 65 00 00 +65 00 dd de 15 a6 65 37 01 00 07 72 00 20 00 00 +66 00 dd de 15 a6 65 37 01 00 08 77 00 69 00 00 +67 00 dd de 15 a6 65 37 01 00 09 6c 00 6c 00 00 +68 00 dd de 15 a6 65 37 01 00 0a 20 00 72 00 00 +69 00 dd de 15 a6 65 37 01 00 0b 65 00 73 00 00 +6a 00 dd de 15 a6 65 37 01 00 0c 74 00 61 00 00 +6b 00 dd de 15 a6 65 37 01 00 0d 72 00 74 00 00 +6c 00 dd de 15 a6 65 37 01 00 0e 20 00 61 00 00 +6d 00 dd de 15 a6 65 37 01 00 0f 74 00 20 00 00 +6e 00 dd de 15 a6 65 37 01 00 10 31 00 2f 00 00 +6f 00 dd de 15 a6 65 37 01 00 11 31 00 36 00 00 +70 00 dd de 15 a6 65 37 01 00 12 2f 00 32 00 00 +71 00 dd de 15 a6 65 37 01 00 13 30 00 32 00 00 +72 00 dd de 15 a6 65 37 01 00 14 34 00 20 00 00 +73 00 dd df 15 a6 65 37 01 00 15 35 00 3a 00 00 +74 00 dd df 15 a6 65 37 01 00 16 33 00 34 00 00 +75 00 dd df 15 a6 65 37 01 00 17 3a 00 31 00 00 +76 00 dd df 15 a6 65 37 01 00 18 33 00 20 00 00 +77 00 dd df 15 a6 65 37 01 00 19 41 00 4d 00 00 +78 00 dd df 15 a6 65 37 01 00 1a 20 00 74 00 00 +79 00 dd df 15 a6 65 37 01 00 1b 6f 00 20 00 00 +7a 00 dd df 15 a6 65 37 01 00 1c 63 00 6f 00 00 +7b 00 dd df 15 a6 65 37 01 00 1d 6d 00 70 00 00 +7c 00 dd df 15 a6 65 37 01 00 1e 6c 00 65 00 00 +7d 00 dd df 15 a6 65 37 01 00 1f 74 00 65 00 00 +7e 00 dd df 15 a6 65 37 01 00 20 20 00 74 00 00 +7f 00 dd df 15 a6 65 37 01 00 21 68 00 65 00 00 +80 00 dd df 15 a6 65 37 01 00 22 20 00 69 00 00 +81 00 dd df 15 a6 65 37 01 00 23 6e 00 73 00 00 +82 00 dd df 15 a6 65 37 01 00 24 74 00 61 00 00 +83 00 dd df 15 a6 65 37 01 00 25 6c 00 6c 00 00 +84 00 dd df 15 a6 65 37 01 00 26 61 00 74 00 00 +85 00 dd df 15 a6 65 37 01 00 27 69 00 6f 00 00 +86 00 dd df 15 a6 65 37 01 00 28 6e 00 20 00 00 +87 00 dd df 15 a6 65 37 01 00 29 6f 00 66 00 00 +88 00 dd df 15 a6 65 37 01 00 2a 20 00 61 00 00 +89 00 dd df 15 a6 65 37 01 00 2b 70 00 70 00 00 +8a 00 dd df 15 a6 65 37 01 00 2c 6c 00 69 00 00 +8b 00 dd df 15 a6 65 37 01 00 2d 63 00 61 00 00 +8c 00 dd df 15 a6 65 37 01 00 2e 74 00 69 00 00 +8d 00 dd df 15 a6 65 37 01 00 2f 6f 00 6e 00 00 +8e 00 dd e0 15 a6 65 37 01 00 30 73 00 20 00 00 +8f 00 dd e0 15 a6 65 37 01 00 31 61 00 6e 00 00 +90 00 dd e0 15 a6 65 37 01 00 32 64 00 20 00 00 +91 00 dd e0 15 a6 65 37 01 00 33 73 00 6f 00 00 +92 00 dd e0 15 a6 65 37 01 00 34 66 00 74 00 00 +93 00 dd e0 15 a6 65 37 01 00 35 77 00 61 00 00 +94 00 dd e0 15 a6 65 37 01 00 36 72 00 65 00 00 +95 00 dd e0 15 a6 65 37 01 00 37 20 00 75 00 00 +96 00 dd e0 15 a6 65 37 01 00 38 70 00 64 00 00 +97 00 dd e0 15 a6 65 37 01 00 39 61 00 74 00 00 +98 00 dd e0 15 a6 65 37 01 00 3a 65 00 73 00 00 +99 00 dd e0 15 a6 65 37 01 00 3b 2e 00 00 00 00 +9a 00 02 ef 15 a6 65 20 00 04 28 ee ef c4 35 ff +9b 00 c8 ef 15 a6 65 9c 9c 00 00 02 00 00 00 00 +9c 00 02 f7 15 a6 65 20 00 04 28 ee 6f c4 35 ff +9d 00 02 1c 16 a6 65 20 00 04 e1 ef 6f 03 00 ff +9e 00 02 1c 16 a6 65 20 00 04 e1 ef 6f 03 10 ff +9f 00 02 1d 16 a6 65 20 00 04 e1 ef 6f 03 20 ff +a0 00 02 1d 16 a6 65 20 00 04 e1 ef 6f 03 40 ff +a1 00 02 59 16 a6 65 20 00 04 e1 ef 6f 03 00 ff +a2 00 02 5a 16 a6 65 20 00 04 e1 ef 6f 03 10 ff +a3 00 02 5a 16 a6 65 20 00 04 e1 ef 6f 03 20 ff +a4 00 02 5a 16 a6 65 20 00 04 e1 ef 6f 03 40 ff +a5 00 02 8f 16 a6 65 41 00 04 1f 00 6f 01 ff ff +a6 00 dc 90 16 a6 65 37 01 00 00 6f 16 a6 65 00 +a7 00 02 a9 92 aa 65 41 00 04 20 00 6f 03 ff ff +a8 00 dd a9 92 aa 65 37 01 00 00 15 00 07 80 00 +a9 00 02 af 92 aa 65 20 00 04 d0 03 6f 84 00 ff +aa 00 02 af 92 aa 65 20 00 04 d0 03 6f 8a 00 ff +ab 00 02 af 92 aa 65 20 00 04 22 39 6f 05 ff ff +ac 00 02 e5 a7 aa 65 20 00 04 08 84 6f 01 48 28 +ad 00 02 e5 a7 aa 65 20 00 04 08 b3 6f 02 02 18 +ae 00 02 e5 a7 aa 65 20 00 04 08 84 6f 05 0e 41 +af 00 02 e6 a7 aa 65 20 00 04 d0 03 6f 80 03 ff +b0 00 02 e6 a7 aa 65 20 00 04 08 84 6f 05 11 41 +b1 00 02 66 21 ae 65 20 00 04 e0 ef 70 a2 01 11 +b2 00 02 66 21 ae 65 20 00 04 08 84 6f 00 ff ff +b3 00 02 66 21 ae 65 20 00 04 29 86 6f 02 ff ff +b4 00 02 68 21 ae 65 20 00 04 08 84 6f 04 aa ff +b5 00 02 68 21 ae 65 20 00 04 08 84 6f 03 01 ff +b6 00 02 68 21 ae 65 20 00 04 08 84 6f 05 80 03 +b7 00 02 69 21 ae 65 20 00 04 08 84 6f 05 00 05 +b8 00 02 82 21 ae 65 20 00 04 22 39 6f 00 ff ff +b9 00 02 85 21 ae 65 20 00 04 23 ca 6f 09 0f 00 +ba 00 c8 86 21 ae 65 9c 9c 00 00 02 00 00 00 00 +bb 00 02 8e 21 ae 65 20 00 04 28 ee 6f c4 35 ff +bc 00 02 a4 21 ae 65 20 00 04 e1 ef 6f 03 00 ff +bd 00 02 a4 21 ae 65 20 00 04 e1 ef 6f 03 10 ff +be 00 02 a4 21 ae 65 20 00 04 e1 ef 6f 03 20 ff +bf 00 02 a4 21 ae 65 20 00 04 e1 ef 6f 03 40 ff +c0 00 02 aa 21 ae 65 20 00 04 e1 ef 6f 03 00 ff +c1 00 02 aa 21 ae 65 20 00 04 e1 ef 6f 03 10 ff +c2 00 02 aa 21 ae 65 20 00 04 e1 ef 6f 03 20 ff +c3 00 02 ab 21 ae 65 20 00 04 e1 ef 6f 03 40 ff +c4 00 02 dc 21 ae 65 41 00 04 1f 00 6f 01 ff ff +c5 00 dc dc 21 ae 65 37 01 00 00 b4 21 ae 65 00 +c6 00 02 ec 8b c4 65 20 00 04 e0 ef 70 a1 01 01 +c7 00 02 8b ef ce 65 20 00 04 28 ee 6f c4 af ff +c8 00 02 bd ef ce 65 20 00 04 28 ee ef c4 af ff +c9 00 02 f7 0f d4 65 41 00 04 20 00 6f 03 ff ff +ca 00 dd f7 0f d4 65 37 01 00 00 ff 00 00 80 00 +cb 00 02 06 10 d4 65 20 00 04 28 ee ef c4 35 ff +cc 00 c8 06 10 d4 65 9c 9c 00 00 02 00 00 00 00 +cd 00 02 0f 10 d4 65 20 00 04 28 ee 6f c4 35 ff +ce 00 02 27 10 d4 65 20 00 04 e1 ef 6f 03 00 ff +cf 00 02 27 10 d4 65 20 00 04 e1 ef 6f 03 10 ff +d0 00 02 28 10 d4 65 20 00 04 e1 ef 6f 03 20 ff +d1 00 02 28 10 d4 65 20 00 04 e1 ef 6f 03 40 ff +d2 00 02 64 10 d4 65 20 00 04 e1 ef 6f 03 00 ff +d3 00 02 64 10 d4 65 20 00 04 e1 ef 6f 03 10 ff +d4 00 02 65 10 d4 65 20 00 04 e1 ef 6f 03 20 ff +d5 00 02 65 10 d4 65 20 00 04 e1 ef 6f 03 40 ff +d6 00 02 9b 10 d4 65 41 00 04 1f 00 6f 01 ff ff +d7 00 dc 9b 10 d4 65 37 01 00 00 78 10 d4 65 00 +d8 00 02 c7 10 d4 65 41 00 04 20 00 6f 03 ff ff +d9 00 dd c7 10 d4 65 37 01 00 00 03 00 02 80 00 +da 00 c8 cd 10 d4 65 9c 9c 00 00 02 00 00 00 00 +db 00 02 cd 10 d4 65 20 00 04 28 ee ef c4 35 ff +dc 00 02 d4 10 d4 65 20 00 04 28 ee 6f c4 35 ff +dd 00 02 ed 10 d4 65 20 00 04 e1 ef 6f 03 00 ff +de 00 02 ee 10 d4 65 20 00 04 e1 ef 6f 03 10 ff +df 00 02 ee 10 d4 65 20 00 04 e1 ef 6f 03 20 ff +e0 00 02 ee 10 d4 65 20 00 04 e1 ef 6f 03 40 ff +e1 00 02 2a 11 d4 65 20 00 04 e1 ef 6f 03 00 ff +e2 00 02 2b 11 d4 65 20 00 04 e1 ef 6f 03 10 ff +e3 00 02 2b 11 d4 65 20 00 04 e1 ef 6f 03 20 ff +e4 00 02 2b 11 d4 65 20 00 04 e1 ef 6f 03 40 ff +e5 00 02 5f 11 d4 65 41 00 04 1f 00 6f 01 ff ff +e6 00 dc 5f 11 d4 65 37 01 00 00 3d 11 d4 65 00 +e7 00 02 2b 11 d4 65 01 00 04 13 a1 6f ab 00 09 +e8 00 02 2b 11 d4 65 01 00 04 07 8b 6f 03 ff ff +e9 00 02 2b 11 d4 65 01 00 04 07 8b 6f 04 ff ff +ea 00 02 2b 11 d4 65 01 00 04 13 a1 6f a7 00 21 +ipmiutil sel, completed successfully +BREAKINGDECODEDLOGS \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_1.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_1.txt new file mode 100644 index 0000000000..3dffc5607e --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_1.txt @@ -0,0 +1,112 @@ +ipmiutil sensor version 3.18 +-- BMC version 18.11, IPMI version 2.0 + ID | SDRType | Type |SNum| Name |Status| Reading +0001 | Full | Fan | 01 | FANS_SWB_PWM | OK | 66.00 % +0005 | Compact | ACPI Power Stat | 05 | Power_Status | Working | +000f | Full | Voltage | 0f | P12V_SCM_VOLT | OK | 12.13 V +0010 | Full | Voltage | 10 | P5V_SCM_VOLT | OK | 5.05 V +0011 | Full | Voltage | 11 | P3V3_SCM_VOLT | OK | 3.31 V +0012 | Full | Voltage | 12 | P3V_BAT_VOLT | OK | 3.18 V +0013 | Full | Temperature | 13 | SCM_INLET_TMP | OK | 24.00 C +0014 | Full | Temperature | 14 | SWB_INLET_TMP | OK | 27.50 C +0016 | Full | Temperature | 16 | SBP_INLET_TMP | OK | 26.50 C +0017 | Full | Temperature | 17 | M2_TMP | OK | 30.00 C +0018 | Full | Temperature | 18 | E1S_1_TMP | OK | 27.00 C +0019 | Full | Temperature | 19 | E1S_2_TMP | OK | 26.00 C +001a | Full | Temperature | 1a | E1S_3_TMP | OK | 27.00 C +001b | Full | Temperature | 1b | E1S_4_TMP | OK | 26.00 C +001c | Full | Temperature | 1c | E1S_5_TMP | OK | 27.00 C +001d | Full | Temperature | 1d | E1S_TMP_MAX | OK | 27.00 C +001e | Full | Temperature | 1e | PDB_OUTLET_TMP | OK | 33.50 C +001f | Full | Power Supply | 1f | HSC0_INPUT_PWR | OK | 115.36 W +0020 | Full | Voltage | 20 | HSC0_INPUT_VOLT | OK | 51.03 V +0021 | Full | Current | 21 | HSC0_OUTPUT_CUR | OK | 2.21 A +0022 | Full | Temperature | 22 | HSC0_TMP | OK | 29.00 C +0023 | Full | Power Supply | 23 | HSC1_INPUT_PWR | OK | 594.00 W +0024 | Full | Voltage | 24 | HSC1_INPUT_VOLT | OK | 51.03 V +0025 | Full | Current | 25 | HSC1_OUTPUT_CUR | OK | 12.35 A +0026 | Full | Temperature | 26 | HSC1_TMP | OK | 24.00 C +0027 | Full | Power Supply | 27 | HSC2_INPUT_PWR | OK | 594.00 W +0028 | Full | Voltage | 28 | HSC2_INPUT_VOLT | OK | 51.34 V +0029 | Full | Current | 29 | HSC2_OUTPUT_CUR | OK | 11.40 A +002a | Full | Temperature | 2a | HSC2_TMP | OK | 29.00 C +002b | Full | Power Supply | 2b | HSC_PWR_SUM | OK | 1342.00 W +002c | Compact | Temperature | 2c | SoC_0_SOCHOT | OK | +002d | Compact | Chip Set | 2d | SoC_ThermalTrip | _ | +0031 | Full | Voltage | 31 | P12V_STBY_VOLT | OK | 11.96 V +0032 | Full | Current | 32 | P12V_STBY_CUR | OK | 6.00 A +0033 | Full | Temperature | 33 | P12V_STBY_TMP | OK | 36.00 C +0034 | Full | Voltage | 34 | P12V_CB1_VOLT | OK | 11.96 V +0035 | Full | Current | 35 | P12V_CB1_CUR | OK | 33.00 A +0036 | Full | Temperature | 36 | P12V_CB1_TMP | OK | 33.00 C +0037 | Full | Voltage | 37 | P12V_CB2_VOLT | OK | 11.96 V +0038 | Full | Current | 38 | P12V_CB2_CUR | OK | 30.00 A +0039 | Full | Temperature | 39 | P12V_CB2_TMP | OK | 36.00 C +003a | Full | Fan | 3a | FAN_0A | OK | 21904.00 RPM +003b | Full | Fan | 3b | FAN_0B | OK | 20655.00 RPM +003c | Full | Fan | 3c | FAN_1A | OK | 21904.00 RPM +003d | Full | Fan | 3d | FAN_1B | OK | 20385.00 RPM +003e | Full | Fan | 3e | FAN_2A | OK | 21756.00 RPM +003f | Full | Fan | 3f | FAN_2B | OK | 20385.00 RPM +0040 | Full | Fan | 40 | FAN_3A | OK | 21904.00 RPM +0041 | Full | Fan | 41 | FAN_3B | OK | 20385.00 RPM +0042 | Full | Fan | 42 | FAN_4A | OK | 21904.00 RPM +0043 | Full | Fan | 43 | FAN_4B | OK | 20385.00 RPM +0044 | Full | Fan | 44 | FAN_5A | OK | 21904.00 RPM +0045 | Full | Fan | 45 | FAN_5B | OK | 20385.00 RPM +0046 | Full | Fan | 46 | FAN_6A | OK | 21904.00 RPM +0047 | Full | Fan | 47 | FAN_6B | OK | 20385.00 RPM +0048 | Full | Fan | 48 | FAN_7A | OK | 21904.00 RPM +0049 | Full | Fan | 49 | FAN_7B | OK | 20385.00 RPM +005a | Full | Temperature | 5a | GPU0_DRAM0_TMP0 | OK | 37.00 C +005b | Full | Temperature | 5b | GPU1_DRAM0_TMP0 | OK | 35.00 C +005c | Full | Temperature | 5c | GPU2_DRAM0_TMP0 | OK | 36.00 C +005d | Full | Temperature | 5d | GPU3_DRAM0_TMP0 | OK | 35.00 C +005e | Full | Temperature | 5e | GPU_DRAM_TMP_MAX | OK | 37.00 C +005f | Full | Power Supply | 5f | GPU0_PWR0 | OK | 215.00 W +0060 | Full | Power Supply | 60 | GPU1_PWR0 | OK | 240.00 W +0061 | Full | Power Supply | 61 | GPU2_PWR0 | OK | 220.00 W +0062 | Full | Power Supply | 62 | GPU3_PWR0 | OK | 210.00 W +0063 | Full | Power Supply | 63 | GPU_PWR_SUM | OK | 880.00 W +0064 | Full | Temperature | 64 | GPU0_TLIMIT_TMP1 | OK | 55.00 C +0065 | Full | Temperature | 65 | GPU1_TLIMIT_TMP1 | OK | 55.00 C +0066 | Full | Temperature | 66 | GPU2_TLIMIT_TMP1 | OK | 55.00 C +0067 | Full | Temperature | 67 | GPU3_TLIMIT_TMP1 | OK | 56.00 C +0068 | Full | Temperature | 68 | GPU_TLIMIT_MIN | OK | 55.00 C +006e | Full | Power Supply | 6e | GPU0_DRAM0_PWR0 | OK | 25.00 W +006f | Full | Power Supply | 6f | GPU1_DRAM0_PWR0 | OK | 25.00 W +0070 | Full | Power Supply | 70 | GPU2_DRAM0_PWR0 | OK | 20.00 W +0071 | Full | Power Supply | 71 | GPU3_DRAM0_PWR0 | OK | 20.00 W +0072 | Full | Power Supply | 72 | GPU_DRAM_PWR_SUM | OK | 100.00 W +0081 | Full | Power Supply | 81 | M0_CPU_PWR0 | OK | 92.00 W +0082 | Full | Power Supply | 82 | M0_VREG_CPUPWR0 | OK | 44.00 W +0083 | Full | Power Supply | 83 | M0_VREG_SOCPWR0 | OK | 6.16 W +0084 | Full | Temperature | 84 | M0_CPU_TMPAVG0 | OK | 35.00 C +0085 | Full | Temperature | 85 | M0_CPU_TMPLIMIT0 | OK | 52.00 C +0086 | Full | Voltage | 86 | M0_VREG_CPUVOLT0 | OK | 0.80 V +0088 | Full | Voltage | 88 | M0_VREG_SOCVOLT0 | OK | 0.80 V +0089 | Full | Power Supply | 89 | M1_CPU_PWR0 | OK | 90.00 W +008a | Full | Power Supply | 8a | M1_VREG_CPUPWR0 | OK | 42.00 W +008b | Full | Power Supply | 8b | M1_VREG_SOCPWR0 | OK | 5.60 W +008c | Full | Temperature | 8c | M1_CPU_TMPAVG0 | OK | 35.00 C +008d | Full | Temperature | 8d | M1_CPU_TMPLIMIT0 | OK | 52.00 C +008e | Full | Voltage | 8e | M1_VREG_CPUVOLT0 | OK | 0.80 V +008f | Full | Voltage | 8f | M1_VREG_SOCVOLT0 | OK | 0.80 V +0090 | Full | Temperature | 90 | OL_S1_HSC_TMP | OK | 22.50 C +0091 | Full | Temperature | 91 | OL_S1_FPGA_TMP | OK | 45.00 C +0092 | Full | Temperature | 92 | OL_S1_AMB_TMP | OK | 38.50 C +0093 | Full | Temperature | 93 | OL_S1_SOC_TMP | OK | 48.00 C +0094 | Full | Power Supply | 94 | OL_S1_POWER | OK | 63.13 W +0095 | Full | Temperature | 95 | PCIESW_DIE_TEMP | OK | 37.00 C +0097 | Full | Temperature | 97 | HMC_TMP | OK | 35.00 C +0098 | Full | Voltage | 98 | LEAK_L_COLDPLATE | OK | 1.74 V +0099 | Full | Voltage | 99 | LEAK_L_MANIFOLD | OK | 1.74 V +009a | Full | Voltage | 9a | LEAK_R_COLDPLATE | OK | 1.74 V +009b | Full | Voltage | 9b | LEAK_R_MANIFOLD | OK | 1.73 V +00b0 | Full | Temperature | b0 | CX7_0_Temp | OK | 44.00 C +00b1 | Full | Temperature | b1 | CX7_1_Temp | OK | 42.00 C +00b2 | Full | Temperature | b2 | CX7_2_Temp | OK | 41.00 C +00b3 | Full | Temperature | b3 | CX7_3_Temp | OK | 40.00 C +00ee | Compact | Management Subs | ee | Subsystem_Health | OK | +00ef | Compact | OEM(e0) | ef | BMC_Health | _ | +ipmiutil sensor, completed successfully \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_2.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_2.txt new file mode 100644 index 0000000000..3c746b5d21 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_2.txt @@ -0,0 +1,163 @@ +Command : sudo ipmiutil sensor -s +WorkingDirectory : +ElapsedTime : 00:00:12.7496509 +StartTime : 2025-09-21T06:47:30.989Z +ExitTime : 2025-09-21T06:47:43.739Z +ExitCode : 0 +ClientId : f4613997-1487-426e-82f8-d4bbcc13fc22 +ComponentType : Monitor +Cycle : 3 +Documentation_CommandLineOptions : https://microsoft.github.io/VirtualClient/docs/guides/0010-command-line/ +experimentDefinitionId : fb71044e-6909-4a91-b06f-e41f29d56656 +ExperimentId : f9ce9a83-87e1-4ed8-8576-f3d55acbcbc6 +experimentName : Linux_OS_Reboot_Rack1 +experimentStepId : 3f7810a5-c9fa-4bdc-9023-ea045d1981d4 +groupId : Group A +MachineName : Rack1-U25 +Notes_FailFastBehavior : The default behavior for Virtual Client is to try to stay alive in the face of errors. To change this behavior and cause Virtual Client to exit with the error code provided by Toolkit, provide the --fail-fast command line option or override the FailFast parameter on the command line. +OperatingSystemDescription : Ubuntu 24.04.2 LTS +OperatingSystemVersion : Unix 6.11.0.1012 +PlatformArchitecture : linux-arm64 +revision : 1.0 +Role : Client +Scenario : CaptureBmcSensorData +SupportedOperatingSystems : CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows +SupportedPlatforms : linux-x64,linux-arm64,win-x64,win-arm64 +TimeZone : Coordinated Universal Time +ToolName : ipmiutil_sensor + +##StandardOutput## +ipmiutil sensor version 3.19 +-- BMC version 19.07, IPMI version 2.0 + ID | SDRType | Type |SNum| Name |Status| Reading +0001 | Full | Fan | 01 | FANS_SWB_PWM | OK | 26.00 % +0005 | Compact | ACPI Power Stat | 05 | Power_Status | Working | +000f | Full | Voltage | 0f | P12V_SCM_VOLT | OK | 12.15 V +0010 | Full | Voltage | 10 | P5V_SCM_VOLT | OK | 5.07 V +0011 | Full | Voltage | 11 | P3V3_SCM_VOLT | OK | 3.32 V +0012 | Full | Voltage | 12 | P3V_BAT_VOLT | OK | 3.34 V +0013 | Full | Temperature | 13 | SCM_INLET_TMP | OK | 26.50 C +0014 | Full | Temperature | 14 | SWB_INLET_TMP | OK | 31.00 C +0016 | Full | Temperature | 16 | SBP_INLET_TMP | OK | 31.00 C +0017 | Full | Temperature | 17 | M2_TMP | OK | 42.00 C +0018 | Full | Temperature | 18 | E1S_1_TMP | OK | 33.00 C +0019 | Full | Temperature | 19 | E1S_2_TMP | OK | 32.00 C +001a | Full | Temperature | 1a | E1S_3_TMP | OK | 33.00 C +001b | Full | Temperature | 1b | E1S_4_TMP | OK | 33.00 C +001c | Full | Temperature | 1c | E1S_5_TMP | OK | 32.00 C +001d | Full | Temperature | 1d | E1S_TMP_MAX | OK | 33.00 C +001e | Full | Temperature | 1e | PDB_OUTLET_TMP | OK | 42.00 C +001f | Full | Power Supply | 1f | HSC0_INPUT_PWR | OK | 120.00 W +0020 | Full | Voltage | 20 | HSC0_INPUT_VOLT | OK | 51.34 V +0021 | Full | Current | 21 | HSC0_OUTPUT_CUR | OK | 2.20 A +0022 | Full | Temperature | 22 | HSC0_TMP | OK | 37.00 C +0023 | Full | Power Supply | 23 | HSC1_INPUT_PWR | OK | 550.00 W +0024 | Full | Voltage | 24 | HSC1_INPUT_VOLT | OK | 51.34 V +0025 | Full | Current | 25 | HSC1_OUTPUT_CUR | OK | 11.00 A +0026 | Full | Temperature | 26 | HSC1_TMP | OK | 33.00 C +0027 | Full | Power Supply | 27 | HSC2_INPUT_PWR | OK | 550.00 W +0028 | Full | Voltage | 28 | HSC2_INPUT_VOLT | OK | 51.03 V +0029 | Full | Current | 29 | HSC2_OUTPUT_CUR | OK | 9.90 A +002a | Full | Temperature | 2a | HSC2_TMP | OK | 38.00 C +002b | Full | Power Supply | 2b | HSC_PWR_SUM | OK | 1236.00 W +002c | Compact | Temperature | 2c | SoC_0_SOCHOT | OK | +002d | Compact | Chip Set | 2d | SoC_ThermalTrip | _ | +0031 | Full | Voltage | 31 | P12V_STBY_VOLT | OK | 11.96 V +0032 | Full | Current | 32 | P12V_STBY_CUR | OK | 8.50 A +0033 | Full | Temperature | 33 | P12V_STBY_TMP | OK | 46.00 C +0034 | Full | Voltage | 34 | P12V_CB1_VOLT | OK | 12.51 V +0035 | Full | Current | 35 | P12V_CB1_CUR | OK | 45.00 A +0036 | Full | Temperature | 36 | P12V_CB1_TMP | OK | 43.00 C +0037 | Full | Voltage | 37 | P12V_CB2_VOLT | OK | 12.51 V +0038 | Full | Current | 38 | P12V_CB2_CUR | OK | 37.50 A +0039 | Full | Temperature | 39 | P12V_CB2_TMP | OK | 46.00 C +003a | Full | Fan | 3a | FAN_0A | OK | 8436.00 RPM +003b | Full | Fan | 3b | FAN_0B | OK | 7290.00 RPM +003c | Full | Fan | 3c | FAN_1A | OK | 8288.00 RPM +003d | Full | Fan | 3d | FAN_1B | OK | 7290.00 RPM +003e | Full | Fan | 3e | FAN_2A | OK | 8288.00 RPM +003f | Full | Fan | 3f | FAN_2B | OK | 7155.00 RPM +0040 | Full | Fan | 40 | FAN_3A | OK | 8288.00 RPM +0041 | Full | Fan | 41 | FAN_3B | OK | 7155.00 RPM +0042 | Full | Fan | 42 | FAN_4A | OK | 8140.00 RPM +0043 | Full | Fan | 43 | FAN_4B | OK | 7290.00 RPM +0044 | Full | Fan | 44 | FAN_5A | OK | 8140.00 RPM +0045 | Full | Fan | 45 | FAN_5B | OK | 7155.00 RPM +0046 | Full | Fan | 46 | FAN_6A | OK | 8288.00 RPM +0047 | Full | Fan | 47 | FAN_6B | OK | 7155.00 RPM +0048 | Full | Fan | 48 | FAN_7A | OK | 8288.00 RPM +0049 | Full | Fan | 49 | FAN_7B | OK | 7155.00 RPM +005a | Full | Temperature | 5a | GPU0_DRAM0_TMP0 | OK | 37.00 C +005b | Full | Temperature | 5b | GPU1_DRAM0_TMP0 | OK | 37.00 C +005c | Full | Temperature | 5c | GPU2_DRAM0_TMP0 | OK | 36.00 C +005d | Full | Temperature | 5d | GPU3_DRAM0_TMP0 | OK | 36.00 C +005e | Full | Temperature | 5e | GPU_DRAM_TMP_MAX | OK | 37.00 C +005f | Full | Power Supply | 5f | GPU0_PWR0 | OK | 180.00 W +0060 | Full | Power Supply | 60 | GPU1_PWR0 | OK | 185.00 W +0061 | Full | Power Supply | 61 | GPU2_PWR0 | OK | 190.00 W +0062 | Full | Power Supply | 62 | GPU3_PWR0 | OK | 185.00 W +0063 | Full | Power Supply | 63 | GPU_PWR_SUM | OK | 740.00 W +0064 | Full | Temperature | 64 | GPU0_TLIMIT_TMP1 | OK | 50.00 C +0065 | Full | Temperature | 65 | GPU1_TLIMIT_TMP1 | OK | 50.00 C +0066 | Full | Temperature | 66 | GPU2_TLIMIT_TMP1 | OK | 50.00 C +0067 | Full | Temperature | 67 | GPU3_TLIMIT_TMP1 | OK | 51.00 C +0068 | Full | Temperature | 68 | GPU_TLIMIT_MIN | OK | 50.00 C +006e | Full | Power Supply | 6e | GPU0_DRAM0_PWR0 | OK | 35.00 W +006f | Full | Power Supply | 6f | GPU1_DRAM0_PWR0 | OK | 35.00 W +0070 | Full | Power Supply | 70 | GPU2_DRAM0_PWR0 | OK | 35.00 W +0071 | Full | Power Supply | 71 | GPU3_DRAM0_PWR0 | OK | 30.00 W +0072 | Full | Power Supply | 72 | GPU_DRAM_PWR_SUM | OK | 140.00 W +0081 | Full | Power Supply | 81 | M0_CPU_PWR0 | OK | 98.00 W +0082 | Full | Power Supply | 82 | M0_VREG_CPUPWR0 | OK | 46.80 W +0083 | Full | Power Supply | 83 | M0_VREG_SOCPWR0 | OK | 6.60 W +0084 | Full | Temperature | 84 | M0_CPU_TMPAVG0 | OK | 41.00 C +0085 | Full | Temperature | 85 | M0_CPU_TMPLIMIT0 | OK | 47.00 C +0086 | Full | Voltage | 86 | M0_VREG_CPUVOLT0 | OK | 0.80 V +0088 | Full | Voltage | 88 | M0_VREG_SOCVOLT0 | OK | 0.80 V +0089 | Full | Power Supply | 89 | M1_CPU_PWR0 | OK | 96.00 W +008a | Full | Power Supply | 8a | M1_VREG_CPUPWR0 | OK | 45.60 W +008b | Full | Power Supply | 8b | M1_VREG_SOCPWR0 | OK | 6.00 W +008c | Full | Temperature | 8c | M1_CPU_TMPAVG0 | OK | 41.00 C +008d | Full | Temperature | 8d | M1_CPU_TMPLIMIT0 | OK | 47.00 C +008e | Full | Voltage | 8e | M1_VREG_CPUVOLT0 | OK | 0.80 V +008f | Full | Voltage | 8f | M1_VREG_SOCVOLT0 | OK | 0.80 V +0090 | Full | Temperature | 90 | OL_S1_HSC_TMP | OK | 32.00 C +0091 | Full | Temperature | 91 | OL_S1_FPGA_TMP | OK | 76.00 C +0092 | Full | Temperature | 92 | OL_S1_AMB_TMP | OK | 64.00 C +0093 | Full | Temperature | 93 | OL_S1_SOC_TMP | OK | 88.00 C +0094 | Full | Power Supply | 94 | OL_S1_POWER | OK | 70.80 W +0095 | Full | Temperature | 95 | PCIESW_DIE_TEMP | OK | 44.00 C +0097 | Full | Temperature | 97 | HMC_TMP | OK | 39.00 C +0098 | Full | Voltage | 98 | LEAK_L_COLDPLATE | OK | 1.72 V +0099 | Full | Voltage | 99 | LEAK_L_MANIFOLD | OK | 1.73 V +009a | Full | Voltage | 9a | LEAK_R_COLDPLATE | OK | 1.73 V +009b | Full | Voltage | 9b | LEAK_R_MANIFOLD | OK | 1.70 V +00a5 | Full | Voltage | a5 | LEAK_L_TRAY | OK | 2.27 V +00a6 | Full | Voltage | a6 | LEAK_R_TRAY | OK | 2.23 V +00a7 | Full | Voltage | a7 | LEAK_Reserved_0 | OK | 3.29 V +00a8 | Full | Voltage | a8 | LEAK_Reserved_1 | OK | 3.26 V +00b0 | Full | Temperature | b0 | CX8_0_Temp | Init | 0.00 C +00b1 | Full | Temperature | b1 | CX8_1_Temp | Init | 0.00 C +00b2 | Full | Temperature | b2 | CX8_2_Temp | Init | 0.00 C +00b3 | Full | Temperature | b3 | CX8_3_Temp | Init | 0.00 C +00b4 | Full | Temperature | b4 | CX8_0_OSFP_Temp | Init | 0.00 C +00b5 | Full | Temperature | b5 | CX8_1_OSFP_Temp | Init | 0.00 C +00b6 | Full | Temperature | b6 | CX8_2_OSFP_Temp | Init | 0.00 C +00b7 | Full | Temperature | b7 | CX8_3_OSFP_Temp | Init | 0.00 C +00bb | Compact | Board | bb | IO_Mezz_Status | OK | +00bc | Compact | OEM(da) | bc | GPUB_GPIO_Status | _ | +00c0 | Full | Temperature | c0 | GPU0_AVG_TMP | OK | 36.00 C +00c1 | Full | Temperature | c1 | GPU1_AVG_TMP | OK | 36.00 C +00c2 | Full | Temperature | c2 | GPU2_AVG_TMP | OK | 36.00 C +00c3 | Full | Temperature | c3 | GPU3_AVG_TMP | OK | 35.00 C +00c4 | Full | Temperature | c4 | GPU_AVG_TMP_MAX | OK | 36.00 C +00c5 | Full | Temperature | c5 | M0_inlet_Temp0 | OK | 38.00 C +00c6 | Full | Temperature | c6 | M0_inlet_Temp1 | OK | 46.00 C +00c7 | Full | Temperature | c7 | M0_Exhaust_Temp0 | OK | 45.00 C +00c8 | Full | Temperature | c8 | M1_inlet_Temp0 | OK | 43.00 C +00c9 | Full | Temperature | c9 | M1_inlet_Temp1 | OK | 47.00 C +00ca | Full | Temperature | ca | M1_Exhaust_Temp0 | OK | 43.00 C +00ee | Compact | Management Subs | ee | Subsystem_Health | OK | +00ef | Compact | OEM(e0) | ef | BMC_Health | _ | +ipmiutil sensor, completed successfully + diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_3.txt b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_3.txt new file mode 100644 index 0000000000..412ba3d602 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Examples/ipmiutil/ipmiutil_sensor_example_3.txt @@ -0,0 +1,112 @@ +ipmiutil sensor version 3.18 +-- BMC version 18.11, IPMI version 2.0 + ID | SDRType | Type |SNum| Name |Status| Reading +0001 | Full | Fan | 01 | FANS_SWB_PWM | OK | NA % +0005 | Compact | ACPI Power Stat | 05 | Power_Status | Working | +000f | Full | Voltage | 0f | P12V_SCM_VOLT | OK | 12.13 V +0010 | Full | Voltage | 10 | P5V_SCM_VOLT | OK | 5.05 V +0011 | Full | Voltage | 11 | P3V3_SCM_VOLT | OK | ... V +0012 | Full | Voltage | 12 | P3V_BAT_VOLT | OK | 3.18 V +0013 | Full | Temperature | 13 | SCM_INLET_TMP | OK | 24.00 C +0014 | Full | Temperature | 14 | SWB_INLET_TMP | OK | 27.50 C +0016 | Full | Temperature | 16 | SBP_INLET_TMP | OK | .. C +0017 | Full | Temperature | 17 | M2_TMP | OK | 30.00 C +0018 | Full | Temperature | 18 | E1S_1_TMP | OK | Na C +0019 | Full | Temperature | 19 | E1S_2_TMP | OK | 26.00 C +001a | Full | Temperature | 1a | E1S_3_TMP | OK | 27.00 C +001b | Full | Temperature | 1b | E1S_4_TMP | OK | 26.00 C +001c | Full | Temperature | 1c | E1S_5_TMP | OK | 27.00 C +001d | Full | Temperature | 1d | E1S_TMP_MAX | OK | 27.00 C +001e | Full | Temperature | 1e | PDB_OUTLET_TMP | OK | 33.50 C +001f | Full | Power Supply | 1f | HSC0_INPUT_PWR | OK | - W +0020 | Full | Voltage | 20 | HSC0_INPUT_VOLT | OK | 51.03 V +0021 | Full | Current | 21 | HSC0_OUTPUT_CUR | OK | N/A A +0022 | Full | Temperature | 22 | HSC0_TMP | OK | 29.00 C +0023 | Full | Power Supply | 23 | HSC1_INPUT_PWR | OK | 594.00 W +0024 | Full | Voltage | 24 | HSC1_INPUT_VOLT | OK | 51.03 V +0025 | Full | Current | 25 | HSC1_OUTPUT_CUR | OK | 12.35 A +0026 | Full | Temperature | 26 | HSC1_TMP | OK | ERR C +0027 | Full | Power Supply | 27 | HSC2_INPUT_PWR | OK | 594.00 W +0028 | Full | Voltage | 28 | HSC2_INPUT_VOLT | OK | 51.34 V +0029 | Full | Current | 29 | HSC2_OUTPUT_CUR | OK | --- A +002a | Full | Temperature | 2a | HSC2_TMP | OK | 29.00 C +002b | Full | Power Supply | 2b | HSC_PWR_SUM | OK | 1342.00 W +002c | Compact | Temperature | 2c | SoC_0_SOCHOT | OK | +002d | Compact | Chip Set | 2d | SoC_ThermalTrip | _ | +0031 | Full | Voltage | 31 | P12V_STBY_VOLT | OK | - V +0032 | Full | Current | 32 | P12V_STBY_CUR | OK | 6.00 A +0033 | Full | Temperature | 33 | P12V_STBY_TMP | OK | 36.00 C +0034 | Full | Voltage | 34 | P12V_CB1_VOLT | OK | 11.96 V +0035 | Full | Current | 35 | P12V_CB1_CUR | OK | 33.00 A +0036 | Full | Temperature | 36 | P12V_CB1_TMP | OK | 33.00 C +0037 | Full | Voltage | 37 | P12V_CB2_VOLT | OK | 11.96 V +0038 | Full | Current | 38 | P12V_CB2_CUR | OK | ?? A +0039 | Full | Temperature | 39 | P12V_CB2_TMP | OK | 36.00 C +003a | Full | Fan | 3a | FAN_0A | OK | 21904.00 RPM +003b | Full | Fan | 3b | FAN_0B | OK | 20655.00 RPM +003c | Full | Fan | 3c | FAN_1A | OK | 21904.00 RPM +003d | Full | Fan | 3d | FAN_1B | OK | .... RPM +003e | Full | Fan | 3e | FAN_2A | OK | 21756.00 RPM +003f | Full | Fan | 3f | FAN_2B | OK | 20385.00 RPM +0040 | Full | Fan | 40 | FAN_3A | OK | 21904.00 RPM +0041 | Full | Fan | 41 | FAN_3B | OK | 20385.00 RPM +0042 | Full | Fan | 42 | FAN_4A | OK | 21904.00 RPM +0043 | Full | Fan | 43 | FAN_4B | OK | 20385.00 RPM +0044 | Full | Fan | 44 | FAN_5A | OK | 21904.00 RPM +0045 | Full | Fan | 45 | FAN_5B | OK | 20385.00 RPM +0046 | Full | Fan | 46 | FAN_6A | OK | UNKNOWN RPM +0047 | Full | Fan | 47 | FAN_6B | OK | 20385.00 RPM +0048 | Full | Fan | 48 | FAN_7A | OK | 21904.00 RPM +0049 | Full | Fan | 49 | FAN_7B | OK | 20385.00 RPM +005a | Full | Temperature | 5a | GPU0_DRAM0_TMP0 | OK | 37.00 C +005b | Full | Temperature | 5b | GPU1_DRAM0_TMP0 | OK | 35.00 C +005c | Full | Temperature | 5c | GPU2_DRAM0_TMP0 | OK | 36.00 C +005d | Full | Temperature | 5d | GPU3_DRAM0_TMP0 | OK | -.- C +005e | Full | Temperature | 5e | GPU_DRAM_TMP_MAX | OK | 37.00 C +005f | Full | Power Supply | 5f | GPU0_PWR0 | OK | 215.00 W +0060 | Full | Power Supply | 60 | GPU1_PWR0 | OK | 240.00 W +0061 | Full | Power Supply | 61 | GPU2_PWR0 | OK | .. W +0062 | Full | Power Supply | 62 | GPU3_PWR0 | OK | 210.00 W +0063 | Full | Power Supply | 63 | GPU_PWR_SUM | OK | 880.00 W +0064 | Full | Temperature | 64 | GPU0_TLIMIT_TMP1 | OK | 55.00 C +0065 | Full | Temperature | 65 | GPU1_TLIMIT_TMP1 | OK | 55.00 C +0066 | Full | Temperature | 66 | GPU2_TLIMIT_TMP1 | OK | 55.00 C +0067 | Full | Temperature | 67 | GPU3_TLIMIT_TMP1 | OK | 56.00 C +0068 | Full | Temperature | 68 | GPU_TLIMIT_MIN | OK | 55.00 C +006e | Full | Power Supply | 6e | GPU0_DRAM0_PWR0 | OK | 25.00 W +006f | Full | Power Supply | 6f | GPU1_DRAM0_PWR0 | OK | 25.00 W +0070 | Full | Power Supply | 70 | GPU2_DRAM0_PWR0 | OK | n/a W +0071 | Full | Power Supply | 71 | GPU3_DRAM0_PWR0 | OK | 20.00 W +0072 | Full | Power Supply | 72 | GPU_DRAM_PWR_SUM | OK | 100.00 W +0081 | Full | Power Supply | 81 | M0_CPU_PWR0 | OK | 92.00 W +0082 | Full | Power Supply | 82 | M0_VREG_CPUPWR0 | OK | 44.00 W +0083 | Full | Power Supply | 83 | M0_VREG_SOCPWR0 | OK | 6.16 W +0084 | Full | Temperature | 84 | M0_CPU_TMPAVG0 | OK | 35.00 C +0085 | Full | Temperature | 85 | M0_CPU_TMPLIMIT0 | OK | 52.00 C +0086 | Full | Voltage | 86 | M0_VREG_CPUVOLT0 | OK | 0.80 V +0088 | Full | Voltage | 88 | M0_VREG_SOCVOLT0 | OK | 0.80 V +0089 | Full | Power Supply | 89 | M1_CPU_PWR0 | OK | 90.00 W +008a | Full | Power Supply | 8a | M1_VREG_CPUPWR0 | OK | 42.00 W +008b | Full | Power Supply | 8b | M1_VREG_SOCPWR0 | OK | 5.60 W +008c | Full | Temperature | 8c | M1_CPU_TMPAVG0 | OK | 35.00 C +008d | Full | Temperature | 8d | M1_CPU_TMPLIMIT0 | OK | 52.00 C +008e | Full | Voltage | 8e | M1_VREG_CPUVOLT0 | OK | INVALID V +008f | Full | Voltage | 8f | M1_VREG_SOCVOLT0 | OK | 0.80 V +0090 | Full | Temperature | 90 | OL_S1_HSC_TMP | OK | 22.50 C +0091 | Full | Temperature | 91 | OL_S1_FPGA_TMP | OK | 45.00 C +0092 | Full | Temperature | 92 | OL_S1_AMB_TMP | OK | 38.50 C +0093 | Full | Temperature | 93 | OL_S1_SOC_TMP | OK | 48.00 C +0094 | Full | Power Supply | 94 | OL_S1_POWER | OK | 63.13 W +0095 | Full | Temperature | 95 | PCIESW_DIE_TEMP | OK | 37.00 C +0097 | Full | Temperature | 97 | HMC_TMP | OK | 35.00 C +0098 | Full | Voltage | 98 | LEAK_L_COLDPLATE | OK | 1.74 V +0099 | Full | Voltage | 99 | LEAK_L_MANIFOLD | OK | 1.74 V +009a | Full | Voltage | 9a | LEAK_R_COLDPLATE | OK | 1.74 V +009b | Full | Voltage | 9b | LEAK_R_MANIFOLD | OK | -- V +00b0 | Full | Temperature | b0 | CX7_0_Temp | OK | 44.00 C +00b1 | Full | Temperature | b1 | CX7_1_Temp | OK | 42.00 C +00b2 | Full | Temperature | b2 | CX7_2_Temp | OK | 41.00 C +00b3 | Full | Temperature | b3 | CX7_3_Temp | OK | 40.00 C +00ee | Compact | Management Subs | ee | Subsystem_Health | OK | +00ef | Compact | OEM(e0) | ef | BMC_Health | _ | +ipmiutil sensor, completed successfully diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMetricsParserTests.cs new file mode 100644 index 0000000000..63286e337a --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMetricsParserTests.cs @@ -0,0 +1,408 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text.RegularExpressions; + using NUnit.Framework; + using VirtualClient.Common.Extensions; + + [TestFixture] + [Category("Unit")] + public class IpmiUtilSelMetricsParserTests : MockFixture + { + private string examplesDirectory; + private string exampleRawResults; + private string exampleDecodedResults; + + [SetUp] + public void SetupTest() + { + this.examplesDirectory = MockFixture.GetDirectory(typeof(IpmiUtilSelMetricsParserTests), "test_examples", "ipmiutil"); + this.exampleRawResults = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "ipmiutil_sel_example_1.txt")); + this.exampleDecodedResults = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "ipmiutil_sel_decoded_example_1.txt")); + } + + [Test] + public void IpmiUtilSelMetricsParserCombinesRawSelEventsWithDecodedSelEventsIntoExpectedFormat() + { + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + + string[] lines = Regex.Split(combinedResults, "\r?\n"); + + Assert.IsNotNull(lines); + Assert.AreEqual(238, lines.Length); + + // Expectation: + // Every record will have the expected column value. Additionally the record ID of the + // decoded data should match the record ID of the raw detail/data (e.g. 0001 == 01 00). + // + // e.g. + // RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail | Evt_raw_detail + // 0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared | 01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff + foreach (var line in lines) + { + Match match = IpmiUtilSelMetricsParser.SelRecordPattern.Match(line); + if (match.Success) + { + Assert.IsTrue(match.Groups.TryGetValue("RecordId", out Group recordId)); + Assert.IsTrue(match.Groups.TryGetValue("Timestamp", out Group timestamp)); + Assert.IsTrue(match.Groups.TryGetValue("Severity", out Group severity)); + Assert.IsTrue(match.Groups.TryGetValue("Source", out Group source)); + Assert.IsTrue(match.Groups.TryGetValue("EventType", out Group eventType)); + Assert.IsTrue(match.Groups.TryGetValue("Sensor", out Group sensor)); + Assert.IsTrue(match.Groups.TryGetValue("EventDetail", out Group eventDetail)); + Assert.IsTrue(match.Groups.TryGetValue("EventRawDetail", out Group eventRawDetail)); + + string decodedRecordId = recordId.Value; + string rawRecordId = string.Join(string.Empty, eventRawDetail.Value.Substring(0, 5).Split(" ").Reverse()); // 01 00 -> 0001 + + Assert.AreEqual(decodedRecordId, rawRecordId); + } + } + } + + [Test] + public void IpmiUtilSelMetricsParserCombinesRawSelEventsWithDecodedSelEventsIntoExpectedFormat_WhenRawResultsAreNotProvided() + { + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(null, this.exampleDecodedResults); + + Assert.AreEqual(this.exampleDecodedResults, combinedResults); + } + + [Test] + public void IpmiUtilSelMetricsParserCombinesRawSelEventsWithDecodedSelEventsIntoExpectedFormat_Record_Order_Does_Not_Matter_1() + { + string[] rawResults = Regex.Split(this.exampleRawResults, @"\r?\n"); + + // Setup: + // The raw results are in reverse order from the decoded results. + List reversedRecords = new List(rawResults.Take(4)); + reversedRecords.AddRange(rawResults.Skip(4).Reverse()); + this.exampleRawResults = string.Join(Environment.NewLine, reversedRecords); + + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + string[] lines = Regex.Split(combinedResults, "\r?\n"); + + Assert.IsNotNull(lines); + Assert.AreEqual(238, lines.Length); + + // Expectation: + // Every record will have the expected column value. Additionally the record ID of the + // decoded data should match the record ID of the raw detail/data (e.g. 0001 == 01 00). + // + // e.g. + // RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail | Evt_raw_detail + // 0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared | 01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff + foreach (var line in lines) + { + Match match = IpmiUtilSelMetricsParser.SelRecordPattern.Match(line); + if (match.Success) + { + Assert.IsTrue(match.Groups.TryGetValue("RecordId", out Group recordId)); + Assert.IsTrue(match.Groups.TryGetValue("Timestamp", out Group timestamp)); + Assert.IsTrue(match.Groups.TryGetValue("Severity", out Group severity)); + Assert.IsTrue(match.Groups.TryGetValue("Source", out Group source)); + Assert.IsTrue(match.Groups.TryGetValue("EventType", out Group eventType)); + Assert.IsTrue(match.Groups.TryGetValue("Sensor", out Group sensor)); + Assert.IsTrue(match.Groups.TryGetValue("EventDetail", out Group eventDetail)); + Assert.IsTrue(match.Groups.TryGetValue("EventRawDetail", out Group eventRawDetail)); + + string decodedRecordId = recordId.Value; + string rawRecordId = string.Join(string.Empty, eventRawDetail.Value.Substring(0, 5).Split(" ").Reverse()); // 01 00 -> 0001 + + Assert.AreEqual(decodedRecordId, rawRecordId); + } + } + } + + [Test] + public void IpmiUtilSelMetricsParserCombinesRawSelEventsWithDecodedSelEventsIntoExpectedFormat_Record_Order_Does_Not_Matter_2() + { + string[] rawResults = Regex.Split(this.exampleRawResults, @"\r?\n"); + + // Setup: + // The raw results are in random order from the decoded results. + List randomizedRecords = new List(rawResults.Take(4)); + randomizedRecords.AddRange(rawResults.Skip(4).Shuffle()); + this.exampleRawResults = string.Join(Environment.NewLine, randomizedRecords); + + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + string[] lines = Regex.Split(combinedResults, "\r?\n"); + + Assert.IsNotNull(lines); + Assert.AreEqual(238, lines.Length); + + // Expectation: + // Every record will have the expected column value. Additionally the record ID of the + // decoded data should match the record ID of the raw detail/data (e.g. 0001 == 01 00). + // + // e.g. + // RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail | Evt_raw_detail + // 0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared | 01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff + foreach (var line in lines) + { + Match match = IpmiUtilSelMetricsParser.SelRecordPattern.Match(line); + if (match.Success) + { + Assert.IsTrue(match.Groups.TryGetValue("RecordId", out Group recordId)); + Assert.IsTrue(match.Groups.TryGetValue("Timestamp", out Group timestamp)); + Assert.IsTrue(match.Groups.TryGetValue("Severity", out Group severity)); + Assert.IsTrue(match.Groups.TryGetValue("Source", out Group source)); + Assert.IsTrue(match.Groups.TryGetValue("EventType", out Group eventType)); + Assert.IsTrue(match.Groups.TryGetValue("Sensor", out Group sensor)); + Assert.IsTrue(match.Groups.TryGetValue("EventDetail", out Group eventDetail)); + Assert.IsTrue(match.Groups.TryGetValue("EventRawDetail", out Group eventRawDetail)); + + string decodedRecordId = recordId.Value; + string rawRecordId = string.Join(string.Empty, eventRawDetail.Value.Substring(0, 5).Split(" ").Reverse()); // 01 00 -> 0001 + + Assert.AreEqual(decodedRecordId, rawRecordId); + } + } + } + + [Test] + public void IpmiUtilSelMetricsParserCombinesRawSelEventsWithDecodedSelEventsIntoExpectedFormat_Record_Order_Does_Not_Matter_3() + { + string[] decodedResults = Regex.Split(this.exampleDecodedResults, @"\r?\n"); + + // Setup: + // The decoded results are in reverse order from the raw results. + List reversedRecords = new List(decodedResults.Take(4)); + reversedRecords.AddRange(decodedResults.Skip(4).Reverse()); + this.exampleDecodedResults = string.Join(Environment.NewLine, reversedRecords); + + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + string[] lines = Regex.Split(combinedResults, "\r?\n"); + + Assert.IsNotNull(lines); + Assert.AreEqual(238, lines.Length); + + // Expectation: + // Every record will have the expected column value. Additionally the record ID of the + // decoded data should match the record ID of the raw detail/data (e.g. 0001 == 01 00). + // + // e.g. + // RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail | Evt_raw_detail + // 0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared | 01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff + foreach (var line in lines) + { + Match match = IpmiUtilSelMetricsParser.SelRecordPattern.Match(line); + if (match.Success) + { + Assert.IsTrue(match.Groups.TryGetValue("RecordId", out Group recordId)); + Assert.IsTrue(match.Groups.TryGetValue("Timestamp", out Group timestamp)); + Assert.IsTrue(match.Groups.TryGetValue("Severity", out Group severity)); + Assert.IsTrue(match.Groups.TryGetValue("Source", out Group source)); + Assert.IsTrue(match.Groups.TryGetValue("EventType", out Group eventType)); + Assert.IsTrue(match.Groups.TryGetValue("Sensor", out Group sensor)); + Assert.IsTrue(match.Groups.TryGetValue("EventDetail", out Group eventDetail)); + Assert.IsTrue(match.Groups.TryGetValue("EventRawDetail", out Group eventRawDetail)); + + string decodedRecordId = recordId.Value; + string rawRecordId = string.Join(string.Empty, eventRawDetail.Value.Substring(0, 5).Split(" ").Reverse()); // 01 00 -> 0001 + + Assert.AreEqual(decodedRecordId, rawRecordId); + } + } + } + + [Test] + public void IpmiUtilSelMetricsParserCombinesRawSelEventsWithDecodedSelEventsIntoExpectedFormat_Record_Order_Does_Not_Matter_4() + { + string[] decodedResults = Regex.Split(this.exampleDecodedResults, @"\r?\n"); + + // Setup: + // The decoded results are in random order from the raw results. + List randomizedRecords = new List(decodedResults.Take(4)); + randomizedRecords.AddRange(decodedResults.Skip(4).Shuffle()); + this.exampleDecodedResults = string.Join(Environment.NewLine, randomizedRecords); + + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + string[] lines = Regex.Split(combinedResults, "\r?\n"); + + Assert.IsNotNull(lines); + Assert.AreEqual(238, lines.Length); + + // Expectation: + // Every record will have the expected column value. Additionally the record ID of the + // decoded data should match the record ID of the raw detail/data (e.g. 0001 == 01 00). + // + // e.g. + // RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail | Evt_raw_detail + // 0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared | 01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff + foreach (var line in lines) + { + Match match = IpmiUtilSelMetricsParser.SelRecordPattern.Match(line); + if (match.Success) + { + Assert.IsTrue(match.Groups.TryGetValue("RecordId", out Group recordId)); + Assert.IsTrue(match.Groups.TryGetValue("Timestamp", out Group timestamp)); + Assert.IsTrue(match.Groups.TryGetValue("Severity", out Group severity)); + Assert.IsTrue(match.Groups.TryGetValue("Source", out Group source)); + Assert.IsTrue(match.Groups.TryGetValue("EventType", out Group eventType)); + Assert.IsTrue(match.Groups.TryGetValue("Sensor", out Group sensor)); + Assert.IsTrue(match.Groups.TryGetValue("EventDetail", out Group eventDetail)); + Assert.IsTrue(match.Groups.TryGetValue("EventRawDetail", out Group eventRawDetail)); + + string decodedRecordId = recordId.Value; + string rawRecordId = string.Join(string.Empty, eventRawDetail.Value.Substring(0, 5).Split(" ").Reverse()); // 01 00 -> 0001 + + Assert.AreEqual(decodedRecordId, rawRecordId); + } + } + } + + [Test] + public void IpmiUtilSelMetricsParserFindsExpectedRecordsToProcess_1() + { + Assert.IsTrue(IpmiUtilSelMetricsParser.TryGetNewRecords(this.exampleDecodedResults, null, out IList newRecords)); + Assert.IsNotEmpty(newRecords); + Assert.IsTrue(newRecords.Count == 234); + } + + [Test] + [TestCase(" 00e8 | 02/19/24 18:42:00")] + [TestCase("00e8 | 02/19/24 18:42:00")] + [TestCase("00e8| 02/19/24 18:42:00")] + [TestCase("00e8|02/19/24 18:42:00")] + [TestCase(" 00e8 | 02/19/24 18:42:00 |")] + [TestCase("00e8 | 02/19/24 18:42:00 | CRT | BMC | Processor | #8b | FRB2 timeout 6f [03 ff ff]")] + public void IpmiUtilSelMetricsParserFindsExpectedRecordsToProcess_2(string lastRecordProcessed) + { + Assert.IsTrue(IpmiUtilSelMetricsParser.TryGetNewRecords(this.exampleDecodedResults, lastRecordProcessed, out IList newRecords)); + Assert.IsNotEmpty(newRecords); + Assert.IsTrue(newRecords.Count == 2); + Assert.IsTrue(newRecords[0].StartsWith("00e9 | 02/19/24 18:42:00")); + Assert.IsTrue(newRecords[1].StartsWith("00ea | 02/19/24 18:42:00")); + } + + [Test] + [TestCase(" 00ea | 02/19/24 18:42:00")] + [TestCase("00ea | 02/19/24 18:42:00")] + [TestCase("00ea| 02/19/24 18:42:00")] + [TestCase("00ea|02/19/24 18:42:00")] + [TestCase("00ea | 02/19/24 18:42:00 | CRT | EFI | Critical Interrupt | #a1 | Bus Warn 6f [a7 00 21]")] + public void IpmiUtilSelMetricsParserFindsExpectedRecordsToProcess_3(string lastRecordProcessed) + { + Assert.IsFalse(IpmiUtilSelMetricsParser.TryGetNewRecords(this.exampleDecodedResults, lastRecordProcessed, out IList newRecords)); + Assert.IsEmpty(newRecords); + } + + [Test] + public void IpmiUtilSelMetricsParserParsesTheExpectedMetricsFromTheResults_1() + { + IpmiUtilSelMetricsParser.ParsingResult result = IpmiUtilSelMetricsParser.Parse(this.exampleDecodedResults); + + Assert.IsNotNull(result); + Assert.IsNotEmpty(result.Metrics); + Assert.IsTrue(result.Metrics.Count == 234); + + // Note: Metric names now include sensor fields in format: ___ + // Verify a few key metric patterns exist + + // Assert at least one "fully-qualified" metric that includes the sensor segment added by this change. + Assert.IsTrue( + result.Metrics.Any(m => Regex.IsMatch(m.Name, @"\bBMC_Event_Log_SEL_Log_Cleared\b", RegexOptions.IgnoreCase)), + "Should contain a full metric name including sensor field (e.g. BMC_Event_Log_SEL_Log_Cleared ...)"); + + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC") && m.Name.Contains("ACPI_Power_State")), "Should contain BMC ACPI Power State events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC") && m.Name.Contains("Battery")), "Should contain BMC Battery events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC") && m.Name.Contains("Event_Log")), "Should contain BMC Event Log events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC") && m.Name.Contains("Power_Supply")), "Should contain BMC Power Supply events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC") && m.Name.Contains("Processor")), "Should contain BMC Processor events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC") && m.Name.Contains("Watchdog")), "Should contain BMC Watchdog events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("EFI") && m.Name.Contains("Critical_Interrupt")), "Should contain EFI Critical Interrupt events"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("Sms") && m.Name.Contains("OS")), "Should contain Sms OS events"); + } + + [Test] + public void IpmiUtilSelMetricsParserParsesTheExpectedMetricsFromTheResults_2() + { + IpmiUtilSelMetricsParser.ParsingResult result = IpmiUtilSelMetricsParser.Parse( + this.exampleDecodedResults, + eventsFilter: new Regex("MAJ|CRT", RegexOptions.IgnoreCase)); + + Assert.IsNotNull(result); + Assert.IsNotEmpty(result.Metrics); + Assert.IsTrue(result.Metrics.Count == 8); + + // Note: Metric names now include sensor fields, so we check for substrings + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC_Power_Supply") && m.Name.Contains("Failure_detected")), "Should contain Power Supply Failure event"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC_Processor") && m.Name.Contains("FRB2")), "Should contain Processor FRB2 event"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC_Processor") && m.Name.Contains("FRB3")), "Should contain Processor FRB3 event"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("BMC_Processor") && m.Name.Contains("IERR")), "Should contain Processor IERR event"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("EFI_Critical_Interrupt") && m.Name.Contains("Bus_Warn")), "Should contain Critical Interrupt Bus Warn event"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("EFI_Critical_Interrupt") && m.Name.Contains("Fatal_NMI")), "Should contain Critical Interrupt Fatal NMI event"); + Assert.IsTrue(result.Metrics.Any(m => m.Name.Contains("EFI_Power_Supply") && m.Name.Contains("Predictive_failure")), "Should contain Power Supply Predictive failure event"); + } + + [Test] + public void IpmiUtilSelMetricsParserIncludesRawHexInMetricNameWhenCombinedRecordsProvided() + { + // Combine raw and decoded results to get records with hex + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + + // Parse the combined results + IpmiUtilSelMetricsParser.ParsingResult result = IpmiUtilSelMetricsParser.Parse(combinedResults); + + Assert.IsNotNull(result); + Assert.IsNotEmpty(result.Metrics); + + // Verify that metrics contain the hex part with underscores + // Format should be: + var metricsWithHex = result.Metrics.Where(m => m.Name.Contains(" ")).ToList(); + Assert.IsNotEmpty(metricsWithHex, "Should have metrics with space-separated hex portion"); + + // Check specific hex patterns - spaces should be replaced with underscores + var metricWithHex = metricsWithHex.FirstOrDefault(); + Assert.IsNotNull(metricWithHex, "Should find at least one metric with hex"); + + // Verify the hex portion format: should have underscores between bytes (e.g., "02_00_02") + string[] parts = metricWithHex.Name.Split(' '); + Assert.AreEqual(2, parts.Length, "Metric name should have text part and hex part separated by space"); + + string hexPart = parts[1]; + Assert.IsTrue(hexPart.Contains("_"), "Hex part should contain underscores between bytes"); + Assert.IsFalse(hexPart.Contains(" "), "Hex part should not contain spaces"); + + // Verify hex pattern: should be pairs of hex digits separated by underscores + Assert.IsTrue(Regex.IsMatch(hexPart, @"^[0-9a-f]{2}(_[0-9a-f]{2})+$", RegexOptions.IgnoreCase), + "Hex part should be in format: XX_XX_XX_... where X is hex digit"); + } + + [Test] + public void IpmiUtilSelMetricsParserMetricNameFormatWithSensorAndHex() + { + // Combine raw and decoded to get full records + string combinedResults = IpmiUtilSelMetricsParser.CombineRecords(this.exampleRawResults, this.exampleDecodedResults); + IpmiUtilSelMetricsParser.ParsingResult result = IpmiUtilSelMetricsParser.Parse(combinedResults); + + // Find a metric that should have all components: source, eventType, sensor, eventDetail, hex + // Looking for records like: "0001 | 12/13/23 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared | 01 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff" + var eventLogMetric = result.Metrics.FirstOrDefault(m => m.Name.Contains("BMC") && m.Name.Contains("Event_Log")); + Assert.IsNotNull(eventLogMetric, "Should find Event Log metric"); + + // Verify format: BMC_Event_Log_SEL_Log_Cleared 01_00_02_87_32_7a_65_20_00_04_10_8a_6f_02_ff_ff + Assert.IsTrue(eventLogMetric.Name.Contains("BMC"), "Should contain source"); + Assert.IsTrue(eventLogMetric.Name.Contains("Event_Log"), "Should contain event type"); + Assert.IsTrue(eventLogMetric.Name.Contains("SEL"), "Should contain sensor"); + Assert.IsTrue(eventLogMetric.Name.Contains("Log_Cleared"), "Should contain event detail"); + + // Verify it has the space separator and hex part + Assert.IsTrue(eventLogMetric.Name.Contains(" "), "Should have space separator between text and hex"); + string[] parts = eventLogMetric.Name.Split(new[] { ' ' }, 2); + Assert.AreEqual(2, parts.Length, "Should have text part and hex part"); + + string hexPart = parts[1]; + // Hex for record 0001 should be: 01_00_02_87_32_7a_65_20_00_04_10_8a_6f_02_ff_ff + Assert.IsTrue(hexPart.StartsWith("01_00"), "Hex part should start with record ID bytes"); + Assert.IsTrue(hexPart.Contains("_"), "Hex bytes should be joined with underscores"); + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs new file mode 100644 index 0000000000..d00781447c --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs @@ -0,0 +1,2440 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Runtime.InteropServices; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class IpmiUtilSelMonitorTests : MockFixture + { + private MockFixture mockFixture; + private DependencyPath mockPackage; + private string examplesDirectory; + private string validSelResults; + + public void SetupTest(PlatformID platform = PlatformID.Win32NT, Architecture architecture = Architecture.X64) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(platform, architecture); + + this.mockFixture.Parameters["ClearSel"] = false; + this.mockFixture.Parameters["EventsFilter"] = "IERR,Correctable Error"; + this.mockFixture.Parameters["PackageName"] = "ipmiutil"; + + // Setup: + // The monitor will get the ipmiutil toolset from a package downloaded by default. + this.mockPackage = new DependencyPath("ipmiutil", this.mockFixture.GetPackagePath("ipmiutil")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + + this.mockFixture.File.Reset(); + this.mockFixture.Directory.Reset(); + + // Setup: + // The ipmiutil sel command produces valid SEL results. + this.examplesDirectory = MockFixture.GetDirectory(typeof(IpmiUtilSelMonitorTests), "test_examples", "ipmiutil"); + this.validSelResults = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "ipmiutil_sel_example_1.txt")); + } + + [Test] + [TestCase(PlatformID.Win32NT, true, "IERR")] + [TestCase(PlatformID.Win32NT, false, "Correctable Error")] + public void IpmiUtilSelMonitorSetsParametersToExpectedValues(PlatformID platform, bool clearSelLog, string eventsFilter) + { + this.SetupTest(platform); + + this.mockFixture.Parameters["ClearSel"] = clearSelLog; + this.mockFixture.Parameters["EventsFilter"] = eventsFilter; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.ClearSel = clearSelLog; + monitor.EventsFilter = eventsFilter; + } + } + + [Test] + [TestCase(true)] + [TestCase(false)] + public void IpmiUtilSelMonitorSetsCaptureSuspectedEventsParameterToExpectedValue(bool captureSuspectedEvents) + { + this.SetupTest(); + this.mockFixture.Parameters["CaptureSuspectedEvents"] = captureSuspectedEvents; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual(captureSuspectedEvents, monitor.CaptureSuspectedEvents); + } + } + + [Test] + [TestCase(true)] + [TestCase(false)] + public void IpmiUtilSelMonitorSetsIncludeWarningsInSuspectedEventsParameterToExpectedValue(bool includeWarnings) + { + this.SetupTest(); + this.mockFixture.Parameters["IncludeWarningsInSuspectedEvents"] = includeWarnings; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual(includeWarnings, monitor.IncludeWarningsInSuspectedEvents); + } + } + + [Test] + public void IpmiUtilSelMonitorCaptureSuspectedEventsDefaultsToFalse() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsFalse(monitor.CaptureSuspectedEvents); + } + } + + [Test] + public void IpmiUtilSelMonitorIncludeWarningsInSuspectedEventsDefaultsToTrue() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsTrue(monitor.IncludeWarningsInSuspectedEvents); + } + } + + [Test] + public void IpmiUtilSelMonitorParseSelRecordReturnsNullForInvalidInput() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsNull(monitor.TestParseSelRecord(null)); + Assert.IsNull(monitor.TestParseSelRecord(string.Empty)); + Assert.IsNull(monitor.TestParseSelRecord("invalid line format")); + } + } + + [Test] + public void IpmiUtilSelMonitorParseSelRecordParsesValidSelRecord() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string validRecord = "07cc | 01/24/25 21:35:33 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00C"; + var record = monitor.TestParseSelRecord(validRecord); + + Assert.IsNotNull(record); + Assert.AreEqual("07cc", record.RecordId); + Assert.AreEqual("01/24/25 21:35:33", record.Timestamp); + Assert.AreEqual("MAJ", record.Severity); + Assert.AreEqual("BMC", record.Source); + Assert.AreEqual("Temperature", record.EventType); + Assert.AreEqual("GPU0_DRAM0_TMP0", record.Sensor); + Assert.AreEqual("Hi Crit thresh actual=127.00C", record.EventDetail); + } + } + + [Test] + [TestCase("CRT", "failure detected", true)] + [TestCase("MAJ", "throttled", true)] + [TestCase("CRT", "power supply", true)] + [TestCase("INF", "normal operation", false)] + [TestCase("WRN", "warning message", false)] + public void IpmiUtilSelMonitorIsSelEventSuspectedDetectsHardwareErrors(string severity, string eventDetail, bool expectedResult) + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + var record = new TestSelRecord + { + Severity = severity, + EventDetail = eventDetail, + EventType = "Hardware", + IsHardwareError = severity == "CRT" || severity == "MAJ" + }; + + bool result = monitor.TestIsSelEventSuspected(record); + Assert.AreEqual(expectedResult, result); + } + } + + [Test] + [TestCase("correctable ecc", true)] + [TestCase("uncorrectable ecc", true)] + [TestCase("ierr", true)] + [TestCase("frb2", true)] + [TestCase("frb3", true)] + [TestCase("predictive failure", true)] + [TestCase("bus error", true)] + [TestCase("fatal nmi", true)] + [TestCase("normal operation", false)] + public void IpmiUtilSelMonitorIsSelEventSuspectedDetectsSuspectedKeywords(string eventDetail, bool expectedResult) + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + var record = new TestSelRecord + { + Severity = "INF", + EventDetail = eventDetail, + EventType = "Hardware", + IsHardwareError = false + }; + + bool result = monitor.TestIsSelEventSuspected(record); + Assert.AreEqual(expectedResult, result); + } + } + + [Test] + [TestCase("asserted", true, true)] + [TestCase("deasserted", true, true)] + [TestCase("exceeded", true, true)] + [TestCase("within", true, true)] + [TestCase("above", true, true)] + [TestCase("below", true, true)] + [TestCase("normal message", true, false)] + [TestCase("asserted", false, false)] + public void IpmiUtilSelMonitorIsSelEventWarningDetectsWarningEvents(string eventDetail, bool includeWarnings, bool expectedResult) + { + this.SetupTest(); + this.mockFixture.Parameters["IncludeWarningsInSuspectedEvents"] = includeWarnings; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + var record = new TestSelRecord + { + Severity = "WRN", + EventDetail = eventDetail, + EventType = "Temperature", + IsHardwareError = false + }; + + bool result = monitor.TestIsSelEventWarning(record); + Assert.AreEqual(expectedResult, result); + } + } + + [Test] + public void IpmiUtilSelMonitorAnalyzeSuspectedSelEventsWithNoEvents() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + var (suspectedEvents, warningEvents) = monitor.TestAnalyzeSuspectedSelEvents(string.Empty); + + Assert.IsNotNull(suspectedEvents); + Assert.IsNotNull(warningEvents); + Assert.AreEqual(0, suspectedEvents.Count); + Assert.AreEqual(0, warningEvents.Count); + } + } + + [Test] + public void IpmiUtilSelMonitorAnalyzeSuspectedSelEventsWithSuspectedEvents() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string selData = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected\n" + + "07cd | 01/24/25 21:35:34 | INF | BMC | System | SEL | normal operation"; + + var (suspectedEvents, warningEvents) = monitor.TestAnalyzeSuspectedSelEvents(selData); + + Assert.AreEqual(1, suspectedEvents.Count); + Assert.IsTrue(suspectedEvents[0].Contains("failure detected")); + Assert.AreEqual(0, warningEvents.Count); + } + } + + [Test] + public void IpmiUtilSelMonitorAnalyzeSuspectedSelEventsWithWarningEvents() + { + this.SetupTest(); + this.mockFixture.Parameters["IncludeWarningsInSuspectedEvents"] = true; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string selData = "07cc | 01/24/25 21:35:33 | WRN | BMC | Temperature | GPU0_DRAM0_TMP0 | threshold exceeded\n" + + "07cd | 01/24/25 21:35:34 | INF | BMC | System | SEL | normal operation"; + + var (suspectedEvents, warningEvents) = monitor.TestAnalyzeSuspectedSelEvents(selData); + + Assert.AreEqual(0, suspectedEvents.Count); + Assert.AreEqual(1, warningEvents.Count); + Assert.IsTrue(warningEvents[0].Contains("threshold exceeded")); + } + } + + [Test] + public void IpmiUtilSelMonitorCountSelRecordsReturnsCorrectCount() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string selData = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected\n" + + "07cd | 01/24/25 21:35:34 | INF | BMC | System | SEL | normal operation\n" + + "Header line that should be ignored\n" + + "07ce | 01/24/25 21:35:35 | MAJ | BMC | Power | PSU1 | power failure"; + + int count = monitor.TestCountSelRecords(selData); + Assert.AreEqual(3, count); + } + } + + [Test] + public void IpmiUtilSelMonitorCountSelRecordsReturnsZeroForEmptyInput() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual(0, monitor.TestCountSelRecords(null)); + Assert.AreEqual(0, monitor.TestCountSelRecords(string.Empty)); + Assert.AreEqual(0, monitor.TestCountSelRecords("No valid SEL records here")); + } + } + + [Test] + [TestCase("CRT", "power failure", true)] + [TestCase("MAJ", "temperature error", true)] + [TestCase("INF", "normal message", false)] + public void IpmiUtilSelMonitorDetermineIfHardwareErrorDetectsHardwareErrors(string severity, string eventDetail, bool expectedResult) + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + bool result = monitor.TestDetermineIfHardwareError(severity, eventDetail); + Assert.AreEqual(expectedResult, result); + } + } + + [Test] + [TestCase("CRT", null, true)] + [TestCase("CRT", "", true)] + [TestCase("CRT", " ", true)] + [TestCase("MAJ", null, true)] + [TestCase("MAJ", "", true)] + [TestCase("MAJ", " ", true)] + [TestCase("INF", null, false)] + [TestCase("INF", "", false)] + [TestCase("INF", " ", false)] + [TestCase("WRN", null, false)] + [TestCase("WRN", "", false)] + [TestCase("WRN", " ", false)] + [TestCase(null, "failure detected", false)] + [TestCase("", "failure detected", false)] + [TestCase(" ", "failure detected", false)] + public void IpmiUtilSelMonitorDetermineIfHardwareErrorHandlesNullOrEmptyInputs(string severity, string eventDetail, bool expectedResult) + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + bool result = monitor.TestDetermineIfHardwareError(severity, eventDetail); + Assert.AreEqual(expectedResult, result); + } + } + + [Test] + public void IpmiUtilSelMonitorWritesSuspectedEventsFileWhenEnabled() + { + this.SetupTest(); + this.mockFixture.Parameters["CaptureSuspectedEvents"] = true; + + List filesWritten = new List(); + List fileContents = new List(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + filesWritten.Add(path); + fileContents.Add(content); + }) + .Returns(Task.CompletedTask); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string selData = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + monitor.TestCaptureSuspectedEventsAsync(selData, EventContext.None, CancellationToken.None).Wait(); + + Assert.IsTrue(filesWritten.Any(f => f.Contains("SuspectSelEntries"))); + Assert.IsTrue(fileContents.Any(c => c.Contains("failure detected"))); + } + } + + [Test] + public void IpmiUtilSelMonitorWritesEmptySuspectedEventsFileWhenNoSuspectedEvents() + { + this.SetupTest(); + this.mockFixture.Parameters["CaptureSuspectedEvents"] = true; + + List filesWritten = new List(); + List fileContents = new List(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + filesWritten.Add(path); + fileContents.Add(content); + }) + .Returns(Task.CompletedTask); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string selData = "07cc | 01/24/25 21:35:33 | INF | BMC | System | SEL | normal operation"; + monitor.TestCaptureSuspectedEventsAsync(selData, EventContext.None, CancellationToken.None).Wait(); + + Assert.IsTrue(filesWritten.Any(f => f.Contains("SuspectSelEntries"))); + Assert.IsTrue(fileContents.Any(c => c.Contains("No suspected SEL events found"))); + } + } + + [Test] + public void IpmiUtilSelMonitorWritesEmptySuspectedEventsFileWhenNoSelData() + { + this.SetupTest(); + this.mockFixture.Parameters["CaptureSuspectedEvents"] = true; + + List filesWritten = new List(); + List fileContents = new List(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + filesWritten.Add(path); + fileContents.Add(content); + }) + .Returns(Task.CompletedTask); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.TestCaptureSuspectedEventsAsync(string.Empty, EventContext.None, CancellationToken.None).Wait(); + + Assert.IsTrue(filesWritten.Any(f => f.Contains("SuspectSelEntries"))); + Assert.IsTrue(fileContents.Any(c => c.Contains("No SEL events found during monitoring period"))); + } + } + + [Test] + public void IpmiUtilSelMonitorWritesWarningEventsFileWhenWarningEventsExist() + { + this.SetupTest(); + this.mockFixture.Parameters["CaptureSuspectedEvents"] = true; + this.mockFixture.Parameters["IncludeWarningsInSuspectedEvents"] = true; + + List filesWritten = new List(); + List fileContents = new List(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + filesWritten.Add(path); + fileContents.Add(content); + }) + .Returns(Task.CompletedTask); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + string selData = "07cc | 01/24/25 21:35:33 | WRN | BMC | Temperature | GPU0_DRAM0_TMP0 | threshold exceeded"; + monitor.TestCaptureSuspectedEventsAsync(selData, EventContext.None, CancellationToken.None).Wait(); + + Assert.IsTrue(filesWritten.Any(f => f.Contains("SuspectSelEntries"))); + Assert.IsTrue(filesWritten.Any(f => f.Contains("WarningSelEntries"))); + Assert.IsTrue(fileContents.Any(c => c.Contains("threshold exceeded"))); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void IpmiUtilSelMonitorHandlesScenariosWhereToolsetPackagesExistButNotForTheParticularPlatformArchitecture(PlatformID expectedPlatform, Architecture expectedArchitecture) + { + this.SetupTest(expectedPlatform, expectedArchitecture); + + // Setup: + // The ipmiutil toolset package exists... + this.SetupToolsetExistsInPackage(); + + // ...but not for the expected platform/architecture folder. + string platformSpecificToolsetLocation = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificToolsetLocation)) + .Returns(false); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + // Expectation: + // An exception should not surface nor should the path have been set to a platform/specific + // path location (as it does not exist). + Assert.DoesNotThrowAsync(() => monitor.InitializeAsync(EventContext.None, CancellationToken.None)); + Assert.IsNull(monitor.IpmiUtilExePath); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public async Task IpmiUtilSelMonitorChecksForTheToolsetInAPackageLocation(PlatformID expectedPlatform, Architecture expectedArchitecture) + { + this.SetupTest(expectedPlatform, expectedArchitecture); + + // Setup: + // The ipmiutil toolset package exists but not for the expected platform/architecture folder. + this.SetupToolsetExistsInPackage(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.InitializeAsync(EventContext.None, CancellationToken.None); + + string expectedToolset = expectedPlatform == PlatformID.Unix ? "ipmiutil" : "ipmiutil.exe"; + + // Expectation: + // The path should be to the location in the dependency package. + string expectedLocation = this.mockFixture.Combine( + this.mockPackage.Path, + this.mockFixture.PlatformArchitectureName, + expectedToolset); + + string actualLocation = monitor.IpmiUtilExePath; + + Assert.AreEqual(expectedLocation, actualLocation); + } + } + + [Test] + [TestCase(0)] + [TestCase(234)] + public async Task IpmiUtilSelMonitorChecksForTheToolsetInACommonLocationDefinedByThePathEnvironmentVariable(int successExitCode) + { + this.SetupTest(); + + // Setup: + // A toolset package containing ipmiutil does not exist on the system. Additionally, the + // ipmiutil toolset does not exist in one of the alternate locations. It does however exist + // in a common location as defined by the PATH environment variable. + this.SetupToolsetPreInstalled(successExitCode); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Expectation: + // The path should be to the alternate location. + string expectedLocation = "ipmiutil.exe"; + string actualLocation = monitor.IpmiUtilExePath; + + Assert.AreEqual(expectedLocation, actualLocation); + } + } + + [Test] + [TestCase(PlatformID.Win32NT, "C:\\BladeFX_latest\\BladeFX\\Tools\\IpmiUtil\\ipmiutil.exe")] + public async Task IpmiUtilSelMonitorChecksForTheToolsetInAlternatePathLocations(PlatformID platform, string expectedLocation) + { + this.SetupTest(platform); + + // Setup: + // A toolset package containing ipmiutil does not exist on the system. However, the + // ipmiutil toolset DOES exist in one of the alternate locations. + this.SetupToolsetExistsInAlternateLocation(expectedLocation); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Expectation: + // The path should be to the alternate location. + string actualLocation = monitor.IpmiUtilExePath; + + Assert.AreEqual(expectedLocation, actualLocation); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void IpmiUtilSelMonitorExitsGracefullyIfTheToolsetCannotBeLocated(PlatformID expectedPlatform, Architecture expectedArchitecture) + { + this.SetupTest(expectedPlatform, expectedArchitecture); + + // Setup: + // A toolset package containing ipmiutil does not exist on the system. Additionally, the + // ipmiutil toolset does not exist in one of the alternate locations. And finally, it does + // not exist in a common location as defined by the PATH environment variable. + // + // ...nowhere to be found!! + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Mock process = new Mock(); + + if ($"{command} {arguments}" == "ipmiutil --help" || $"{command} {arguments}" == "ipmiutil.exe /?") + { + process.Setup(command, arguments, workingDir, exitCode: 1); + } + else + { + Assert.Fail("Code flow should not proceed if the ipmiutil toolset path cannot be located."); + } + + return process.Object; + }; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + // Expectation: + // The monitor should exit gracefully and there should not be any path location + // identified for the ipmiutil toolset. + + Assert.DoesNotThrowAsync(() => monitor.ExecuteAsync(CancellationToken.None)); + Assert.IsNull(monitor.IpmiUtilExePath); + } + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task IpmiUtilSelMonitorExecutesTheExpectedToolsetCommands_Toolset_Package_Downloaded(PlatformID platform, Architecture architecture) + { + using (CancellationTokenSource tokenSource = new CancellationTokenSource()) + { + List commandsExecuted = new List(); + this.SetupTest(platform, architecture); + + // Setup: + // A package containing ipmiutil is downloaded to the system. + this.SetupToolsetExistsInPackage(); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + commandsExecuted.Add($"{command} {arguments}"); + InMemoryProcess process = new InMemoryProcess(); + + if (arguments.EndsWith("sel -uc")) + { + process.StandardOutput.Append("0002 | 12/13/24 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared"); + } + else if (arguments.EndsWith("sel -r")) + { + process.StandardOutput.Append("02 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff"); + + // Cause the 'while' loop in the monitor to exit. + tokenSource.Cancel(); + } + + return process; + }; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await Task.WhenAny(monitor.ExecuteAsync(tokenSource.Token), Task.Delay(200000)); + } + + string expectedPackagePath = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + if (platform == PlatformID.Win32NT) + { + Assert.AreEqual(2, commandsExecuted.Count); + Assert.AreEqual($"{expectedPackagePath}\\ipmiutil.exe sel -uc", commandsExecuted[0]); + Assert.AreEqual($"{expectedPackagePath}\\ipmiutil.exe sel -r", commandsExecuted[1]); + } + else if (platform == PlatformID.Unix) + { + Assert.AreEqual(3, commandsExecuted.Count); + Assert.AreEqual($"sudo chmod +x \"{expectedPackagePath}/ipmiutil\"", commandsExecuted[0]); + Assert.AreEqual($"sudo {expectedPackagePath}/ipmiutil sel -uc", commandsExecuted[1]); + Assert.AreEqual($"sudo {expectedPackagePath}/ipmiutil sel -r", commandsExecuted[2]); + } + } + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task IpmiUtilSelMonitorExecutesTheExpectedToolsetCommands_Toolset_PreInstalled(PlatformID platform, Architecture architecture) + { + using (CancellationTokenSource tokenSource = new CancellationTokenSource()) + { + List commandsExecuted = new List(); + this.SetupTest(platform, architecture); + + // Setup: + // A package containing ipmiutil is NOT installed. This will force the logic to attempt to + // identify the binary installation location in other locations. + this.SetupToolsetPreInstalled(0); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + commandsExecuted.Add($"{command} {arguments}"); + InMemoryProcess process = new InMemoryProcess(); + + // Setup: + // The toolset is preinstalled on the system (e.g. via Linux package manager). + if (arguments.EndsWith("--help") || arguments.EndsWith("/?")) + { + process.ExitCode = 0; + process.StandardOutput.Append("ipmiutil ver. 1.2.3"); + } + else if (arguments.EndsWith("sel -uc")) + { + process.StandardOutput.Append("0002 | 12/13/24 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared"); + } + else if (arguments.EndsWith("sel -r")) + { + process.StandardOutput.Append("02 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff"); + + // Cause the 'while' loop in the monitor to exit. + tokenSource.Cancel(); + } + + return process; + }; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await Task.WhenAny(monitor.ExecuteAsync(tokenSource.Token), Task.Delay(200000)); + } + + string expectedPackagePath = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + if (platform == PlatformID.Win32NT) + { + Assert.AreEqual(3, commandsExecuted.Count); + Assert.AreEqual($"ipmiutil.exe /?", commandsExecuted[0]); + Assert.AreEqual($"ipmiutil.exe sel -uc", commandsExecuted[1]); + Assert.AreEqual($"ipmiutil.exe sel -r", commandsExecuted[2]); + } + else if (platform == PlatformID.Unix) + { + Assert.AreEqual(3, commandsExecuted.Count); + Assert.AreEqual($"ipmiutil --help", commandsExecuted[0]); + Assert.AreEqual($"sudo ipmiutil sel -uc", commandsExecuted[1]); + Assert.AreEqual($"sudo ipmiutil sel -r", commandsExecuted[2]); + } + } + } + + [Test] + [TestCase(PlatformID.Win32NT, "C:\\BladeFX_latest\\BladeFX\\Tools\\IpmiUtil\\ipmiutil.exe")] + public async Task IpmiUtilSelMonitorExecutesTheExpectedToolsetCommands_Toolset_Installed_In_Alternate_Location(PlatformID platform, string expectedLocation) + { + using (CancellationTokenSource tokenSource = new CancellationTokenSource()) + { + List commandsExecuted = new List(); + this.SetupTest(platform); + + // Setup: + // A package containing ipmiutil is NOT installed. This will force the logic to attempt to + // identify the binary installation location in other locations. The toolset is installed in + // an alternate location on the system. + this.SetupToolsetExistsInAlternateLocation(expectedLocation); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + commandsExecuted.Add($"{command} {arguments}"); + InMemoryProcess process = new InMemoryProcess(); + + // Setup: + // The toolset is not preinstalled on the system (e.g. via Linux package manager). + if (arguments.EndsWith("--help") || arguments.EndsWith("/?")) + { + process.ExitCode = 1; + } + else if (arguments.EndsWith("sel -uc")) + { + process.StandardOutput.Append("0002 | 12/13/24 14:39:03 | INF | BMC| Event Log | SEL | Log Cleared"); + } + else if (arguments.EndsWith("sel -r")) + { + process.StandardOutput.Append("02 00 02 87 32 7a 65 20 00 04 10 8a 6f 02 ff ff"); + + // Cause the 'while' loop in the monitor to exit. + tokenSource.Cancel(); + } + + return process; + }; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await Task.WhenAny(monitor.ExecuteAsync(tokenSource.Token), Task.Delay(200000)); + } + + string expectedPackagePath = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + Assert.AreEqual(3, commandsExecuted.Count); + Assert.AreEqual($"{expectedLocation} sel -uc", commandsExecuted[1]); + Assert.AreEqual($"{expectedLocation} sel -r", commandsExecuted[2]); + } + } + + [Test] + public void IpmiUtilSelMonitorEnableSelDecodingDefaultsToFalse() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsFalse(monitor.EnableSelDecoding); + } + } + + [Test] + [TestCase(true)] + [TestCase(false)] + public void IpmiUtilSelMonitorSetsEnableSelDecodingParameterToExpectedValue(bool enableSelDecoding) + { + this.SetupTest(); + this.mockFixture.Parameters["EnableSelDecoding"] = enableSelDecoding; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual(enableSelDecoding, monitor.EnableSelDecoding); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderArgumentsReturnsNullWhenNotProvided() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsNull(monitor.DecoderArguments); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderArgumentsReturnsValueWhenProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderArguments"] = "-nc -t Pacific -v --no-animation"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual("-nc -t Pacific -v --no-animation", monitor.DecoderArguments); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderPackageNameReturnsNullWhenNotProvided() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsNull(monitor.DecoderPackageName); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderPackageNameReturnsValueWhenProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderPackageName"] = "sherlockSelDecoder"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual("sherlockSelDecoder", monitor.DecoderPackageName); + } + } + + [Test] + public async Task IpmiUtilSelMonitorInitializesSherlockToolsetWhenPackageIsAvailable() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Sherlock"; + this.mockFixture.Parameters["DecoderPackageName"] = "sherlockSelDecoder"; + this.SetupSherlockPackage(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNotNull(monitor.DecoderExePath); + StringAssert.Contains("sel_decode.exe", monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorSetsDecoderExePathToNullWhenPackageNameIsNotProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Sherlock"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNull(monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorSetsDecoderExePathToNullWhenExecutableDoesNotExistInPackage() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Sherlock"; + this.mockFixture.Parameters["DecoderPackageName"] = "sherlockSelDecoder"; + + var sherlockPackage = new DependencyPath("sherlockSelDecoder", this.mockFixture.GetPackagePath("sherlockSelDecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(sherlockPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(sherlockPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(sherlockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + // sel_decode.exe does not exist at the expected path + this.mockFixture.File + .Setup(file => file.Exists(It.Is(s => s.Contains("sel_decode.exe")))) + .Returns(false); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNull(monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorSherlockDecodingSkipsWhenSelRecordsIsEmpty() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + // Should not throw and should skip decoding + await monitor.TestExecuteSherlockDecodingAsync(string.Empty, EventContext.None, CancellationToken.None); + await monitor.TestExecuteSherlockDecodingAsync(null, EventContext.None, CancellationToken.None); + } + } + + [Test] + public async Task IpmiUtilSelMonitorSherlockDecodingWritesSelRecordsToFileAndExecutesDecoder() + { + this.SetupTest(); + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + string writtenContent = null; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + if (path.Contains("selRecords")) + { + writtenContent = content; + } + }) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.AreEqual(selRecords, writtenContent); + } + + [Test] + public async Task IpmiUtilSelMonitorSherlockDecodingPassesDecoderArgumentsToDecoder() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderArguments"] = "-nc -t Pacific -v --no-animation"; + + string capturedArguments = null; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + capturedArguments = arguments; + return new InMemoryProcess(); + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.IsNotNull(capturedArguments); + StringAssert.Contains("-nc -t Pacific -v --no-animation", capturedArguments); + StringAssert.Contains("-f", capturedArguments); + } + + [Test] + public void IpmiUtilSelMonitorSherlockDecodingDoesNotThrowOnNonZeroExitCode() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.ExitCode = 1; + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + // Should not throw - non-zero exit code is logged as warning, not thrown + Assert.DoesNotThrowAsync(() => monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None)); + } + } + + [Test] + public async Task IpmiUtilSelMonitorSherlockDecodingOmitsExtraArgsWhenDecoderArgumentsIsNull() + { + this.SetupTest(); + + string capturedArguments = null; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + capturedArguments = arguments; + return new InMemoryProcess(); + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.IsNotNull(capturedArguments); + StringAssert.StartsWith(" -f", capturedArguments); + } + + [Test] + public void IpmiUtilSelMonitorSherlockInitializationDoesNotThrowOnException() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Sherlock"; + this.mockFixture.Parameters["DecoderPackageName"] = "sherlockSelDecoder"; + + // Setup: + // Force GetPlatformSpecificPackageAsync to throw an exception by returning a package + // but having the file system throw when checking file existence. + var sherlockPackage = new DependencyPath("sherlockSelDecoder", this.mockFixture.GetPackagePath("sherlockSelDecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(sherlockPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(sherlockPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(sherlockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(It.Is(s => s.Contains("sel_decode.exe")))) + .Throws(new InvalidOperationException("Simulated initialization error")); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + // Expectation: + // The outer catch block in InitializeDecoderAsync should catch the exception + // and log it as a warning without throwing. + Assert.DoesNotThrowAsync(() => monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None)); + Assert.IsNull(monitor.DecoderExePath); + } + } + + [Test] + public void IpmiUtilSelMonitorSherlockDecodingDoesNotThrowOnFileWriteException() + { + this.SetupTest(); + + // Setup: + // Force an exception during file write operations. + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new IOException("Simulated file write error")); + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + // Expectation: + // The catch block in ExecuteSherlockDecodingAsync should catch the exception + // and log it as a warning without throwing. + Assert.DoesNotThrowAsync(() => monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None)); + } + } + + [Test] + public void IpmiUtilSelMonitorSherlockDecodingDoesNotThrowOnProcessException() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Setup: + // Force an exception when creating the process. + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + throw new InvalidOperationException("Simulated process execution error"); + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + // Expectation: + // The catch block in ExecuteSherlockDecodingAsync should catch the exception + // and log it as a warning without throwing. + Assert.DoesNotThrowAsync(() => monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None)); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesAndLogsDecodedSelEntries_WithValidOutput() + { + this.SetupTest(); + + string decodedSelOutput = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "decodedIpmiUtilSelOutput.txt")); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.Greater(monitor.DecodedSelCount, 0, "Expected at least one SEL entry to be decoded."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesAndLogsDecodedSelEntries_WithMultiLineMessages() + { + this.SetupTest(); + + string decodedSelOutput = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "decodedIpmiUtilSelOutput.txt")); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.Greater(monitor.DecodedSelCount, 2, "Expected at least 3 SEL entries to be decoded."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesAndLogsDecodedSelEntries_ExtractsCorrectFields() + { + this.SetupTest(); + + string decodedSelOutput = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "decodedIpmiUtilSelOutput.txt")); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.Greater(monitor.DecodedSelCount, 0, "Expected SEL entries to be decoded."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesAndLogsDecodedSelEntries_SkipsWhenNoTableFound() + { + this.SetupTest(); + + string decodedSelOutput = "Some random output without a table"; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(0, monitor.DecodedSelCount, "No SEL entries should be decoded when table is not found."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesAndLogsDecodedSelEntries_HandlesParsingErrors() + { + this.SetupTest(); + + string decodedSelOutput = @" +ID Timestamp (UTC) Sensor/OEM(Owner) Sensor Type Event Type Message Direction Record Type Raw Hex +---- ------------------- --------------------------------------------- -------------------------------- ------------------------ -------------------------------------------------- ---------- --------------- ------------------------------------------------ +0x1 2026-03-19 19:39:40 SEL(BMC)=0x03 +0x2 2026-03-19 19:41:28 SoC_Status(BMC)=0x98 BMC OEM OEM Discrete POST Complete Timeout ASSERTED System Event 02 00 02 68 51 bc 69 20 00 04 df 98 70 aa e3 ff +"; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(1, monitor.DecodedSelCount, "Only valid SEL entry should be decoded."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorLogsDecodedSelCountMetric() + { + this.SetupTest(); + + string decodedSelOutput = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "decodedIpmiUtilSelOutput.txt")); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.Greater(monitor.DecodedSelCount, 0, "DecodedSelCount should be greater than 0."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesMultipleDecodedSelEntries() + { + this.SetupTest(); + + string decodedSelOutput = System.IO.File.ReadAllText(Path.Combine(this.examplesDirectory, "decodedIpmiUtilSelOutput.txt")); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.StandardOutput.Append(decodedSelOutput); + return process; + }; + + string selRecords = "07cc | 01/24/25 21:35:33 | CRT | BMC | Temperature | GPU0_DRAM0_TMP0 | failure detected"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "sel_decode.exe"; + + await monitor.TestExecuteSherlockDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.Greater(monitor.DecodedSelCount, 5, "Expected multiple SEL entries to be decoded."); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderTypeReturnsValueWhenProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Generic"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual("Generic", monitor.DecoderType); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderExecutableNameReturnsNullWhenNotProvided() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.IsNull(monitor.DecoderExecutableName); + } + } + + [Test] + public void IpmiUtilSelMonitorDecoderExecutableNameReturnsValueWhenProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderExecutableName"] = "custom_decoder.exe"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.AreEqual("custom_decoder.exe", monitor.DecoderExecutableName); + } + } + + [Test] + public async Task IpmiUtilSelMonitorInitializesGenericDecoderToolsetWhenPackageIsAvailable() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Generic"; + this.mockFixture.Parameters["DecoderPackageName"] = "crcsdk_seldecoder"; + this.SetupGenericPackage(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNotNull(monitor.DecoderExePath); + StringAssert.Contains("crcsdkseldecoder.exe", monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorInitializesGenericDecoderToolsetWhenPackageIsAvailableOnLinux() + { + this.SetupTest(PlatformID.Unix, Architecture.X64); + this.mockFixture.Parameters["DecoderType"] = "Generic"; + this.mockFixture.Parameters["DecoderPackageName"] = "crcsdk_seldecoder"; + + var genericPackage = new DependencyPath("crcsdk_seldecoder", this.mockFixture.GetPackagePath("crcsdk_seldecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(genericPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(genericPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(genericPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(this.mockFixture.Combine(platformSpecificLocation, "crcsdkseldecoder"))) + .Returns(true); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNotNull(monitor.DecoderExePath); + StringAssert.Contains("crcsdkseldecoder", monitor.DecoderExePath); + StringAssert.DoesNotContain(".exe", monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericInitializationSetsDecoderExePathToNullWhenPackageNameIsNotProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Generic"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNull(monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericInitializationSetsDecoderExePathToNullWhenExecutableDoesNotExistInPackage() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Generic"; + this.mockFixture.Parameters["DecoderPackageName"] = "crcsdk_seldecoder"; + + var genericPackage = new DependencyPath("crcsdk_seldecoder", this.mockFixture.GetPackagePath("crcsdk_seldecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(genericPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(genericPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(genericPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + // crcsdkseldecoder.exe does not exist at the expected path + this.mockFixture.File + .Setup(file => file.Exists(It.Is(s => s.Contains("crcsdkseldecoder.exe")))) + .Returns(false); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNull(monitor.DecoderExePath); + } + } + + [Test] + public void IpmiUtilSelMonitorInitializeDecoderThrowsArgumentExceptionForUnsupportedDecoderType() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "InvalidDecoder"; + this.mockFixture.Parameters["DecoderPackageName"] = "crcsdk_seldecoder"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + Assert.ThrowsAsync( + () => monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None)); + } + } + + [Test] + public async Task IpmiUtilSelMonitorInitializeDecoderUsesCustomExecutableNameWhenProvided() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderType"] = "Generic"; + this.mockFixture.Parameters["DecoderPackageName"] = "crcsdk_seldecoder"; + this.mockFixture.Parameters["DecoderExecutableName"] = "custom_decoder.exe"; + + var genericPackage = new DependencyPath("crcsdk_seldecoder", this.mockFixture.GetPackagePath("crcsdk_seldecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(genericPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(genericPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(genericPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(this.mockFixture.Combine(platformSpecificLocation, "custom_decoder.exe"))) + .Returns(true); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + await monitor.TestInitializeDecoderAsync(EventContext.None, CancellationToken.None); + + Assert.IsNotNull(monitor.DecoderExePath); + StringAssert.Contains("custom_decoder.exe", monitor.DecoderExePath); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingSkipsWhenSelRecordsIsEmpty() + { + this.SetupTest(); + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + // Should not throw and should skip decoding + await monitor.TestExecuteGenericDecodingAsync(string.Empty, EventContext.None, CancellationToken.None); + await monitor.TestExecuteGenericDecodingAsync(null, EventContext.None, CancellationToken.None); + + Assert.AreEqual(0, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingWritesSelRecordsToFileAndExecutesDecoder() + { + this.SetupTest(); + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + string writtenContent = null; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + if (path.Contains("selRecords")) + { + writtenContent = content; + } + }) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.AreEqual(selRecords, writtenContent); + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingPassesDecoderArgumentsToDecoder() + { + this.SetupTest(); + this.mockFixture.Parameters["DecoderArguments"] = "--sku c41a8"; + + string capturedArguments = null; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + capturedArguments = arguments; + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.IsNotNull(capturedArguments); + StringAssert.Contains("--sku c41a8", capturedArguments); + StringAssert.Contains("-o", capturedArguments); + // Generic format: "{inputFile}" -o "{outputDir}" {DecoderArguments} — file comes first + StringAssert.StartsWith("\"", capturedArguments); + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingOmitsExtraArgsWhenDecoderArgumentsIsNull() + { + this.SetupTest(); + + string capturedArguments = null; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + capturedArguments = arguments; + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.IsNotNull(capturedArguments); + StringAssert.Contains("-o", capturedArguments); + StringAssert.StartsWith("\"", capturedArguments); + } + + [Test] + public void IpmiUtilSelMonitorGenericDecodingDoesNotThrowOnNonZeroExitCode() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + InMemoryProcess process = new InMemoryProcess(); + process.ExitCode = 1; + return process; + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + // Should not throw - non-zero exit code is logged as warning, not thrown + Assert.DoesNotThrowAsync(() => monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None)); + Assert.AreEqual(0, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingHandlesEmptyDecodedOutputFile() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(string.Empty); + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(0, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingHandlesMissingDecodedOutputFile() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(false); + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(0, monitor.DecodedSelCount); + } + } + + [Test] + public void IpmiUtilSelMonitorGenericDecodingDoesNotThrowOnFileWriteException() + { + this.SetupTest(); + + // Setup: + // Force an exception during file write operations. + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new IOException("Simulated file write error")); + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + // Expectation: + // The catch block in ExecuteGenericDecodingAsync should catch the exception + // and log it as a warning without throwing. + Assert.DoesNotThrowAsync(() => monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None)); + } + } + + [Test] + public void IpmiUtilSelMonitorGenericDecodingDoesNotThrowOnProcessException() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Setup: + // Force an exception when creating the process. + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + throw new InvalidOperationException("Simulated process execution error"); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + // Expectation: + // The catch block in ExecuteGenericDecodingAsync should catch the exception + // and log it as a warning without throwing. + Assert.DoesNotThrowAsync(() => monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None)); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingUsesMillisecondGranularityInFileNames() + { + this.SetupTest(); + + string capturedInputPath = null; + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + if (path.Contains("selRecords")) + { + capturedInputPath = path; + } + }) + .Returns(Task.CompletedTask); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + } + + Assert.IsNotNull(capturedInputPath); + // Millisecond format: yyyy-MM-dd-HH-mm-ss-fff — the timestamp portion has 23 chars + // Example: 2026-05-17-10-27-15-830_input_selRecords.txt + string fileName = Path.GetFileName(capturedInputPath); + string timestampPart = fileName.Substring(0, fileName.IndexOf("_input_selRecords")); + Assert.AreEqual(23, timestampPart.Length, "Timestamp should be in yyyy-MM-dd-HH-mm-ss-fff format (23 chars)."); + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_WithSerRecords() + { + this.SetupTest(); + + // SER record format (7 fields): RecordID | Timestamp | Source | SensorType | SensorID | Message | Direction + string decodedOutput = string.Join( + Environment.NewLine, + "0001 | [11/05/2025 18:56:57] | BMC, LUN 0 | Event Logging Disabled | ID:0xF1 | Log Area Reset/cleared | Asserted", + "0009 | [11/11/2025 23:42:10] | BMC, LUN 0 | Physical Security (Chassis Intrusion) | ID:0xF2 | General Chassis Intrusion. detected at power off state | Asserted"); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(2, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_WithOemTimestampedRecords() + { + this.SetupTest(); + + // OEM Timestamped format (5 fields): RecordID | Timestamp | Manufacturer | RecordName | Message + string decodedOutput = string.Join( + Environment.NewLine, + "0007 | [11/11/2025 23:41:59] | BMC | Generic OEM SEL | Component ID: Manticore. Status: Keep Alive normal", + "0015 | [11/11/2025 23:42:24] | Microsoft | RecType: 0xCA | Unknown OEM record type"); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(2, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_WithOemNonTimestampedRecords() + { + this.SetupTest(); + + // OEM Non-timestamped format (4 fields): RecordID | Manufacturer | RecordName | Message + string decodedOutput = "00F0 | Microsoft | OEM_NonTS | Raw OEM data"; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(1, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_WithMixedRecordTypes() + { + this.SetupTest(); + + // Mix of SER (7 fields), OEM Timestamped (5 fields), and OEM Non-timestamped (4 fields) + string decodedOutput = string.Join( + Environment.NewLine, + "0001 | [11/05/2025 18:56:57] | BMC, LUN 0 | Event Logging Disabled | ID:0xF1 | Log Area Reset/cleared | Asserted", + "0007 | [11/11/2025 23:41:59] | BMC | Generic OEM SEL | Component ID: Manticore. Status: Keep Alive normal", + "00F0 | Microsoft | OEM_NonTS | Raw OEM data"); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(3, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_ReturnsZeroForEmptyOutput() + { + this.SetupTest(); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(" \n \n"); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(0, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_CountsLinesWithFewerThanFourFieldsAsUnknown() + { + this.SetupTest(); + + // Lines with fewer than 4 pipe-delimited fields fall into the Unknown fallback path + string decodedOutput = string.Join( + Environment.NewLine, + "some random text", + "0001 | partial | data", + "0002 | [11/05/2025 18:56:57] | BMC, LUN 0 | Event Logging Disabled | ID:0xF1 | Log Area Reset/cleared | Asserted"); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(3, monitor.DecodedSelCount, "All lines should be counted: 2 with fewer than 4 fields logged as Unknown, plus 1 valid SER record."); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_PreservesPipeCharactersInMessageField() + { + this.SetupTest(); + + // SER record where the message itself contains pipe characters — string.Join("|", ...) should + // recombine the middle parts into the message, preserving the original content. + string decodedOutput = "0001 | [11/05/2025 18:56:57] | BMC, LUN 0 | Event Logging Disabled | ID:0xF1 | Message with | embedded | pipes | Asserted"; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(1, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorParsesGenericDecodedSelEntries_PreservesUnicodeCharacters() + { + this.SetupTest(); + + // OEM records containing CJK/Unicode characters in the message field. + // \u6800 = 栀, \u5400 = 吀, \u6500 = 攀, \u7900 = 礀, \u7300 = 猀 + string decodedOutput = string.Join( + Environment.NewLine, + "0001 | [05/12/2026 20:05:11] | BMC, LUN 0 | Event Logging Disabled | ID:0xF1 | Log Area Reset/cleared | Asserted", + "0004 | [05/12/2026 20:05:11] | Microsoft | OS Shutdown | Sequence Number: 1. Shutdown Comment: \u6800\u5400", + "0005 | [05/12/2026 20:05:11] | Microsoft | OS Shutdown | Sequence Number: 1. Shutdown Comment: \u6500", + "0006 | [05/12/2026 20:05:11] | Microsoft | OS Shutdown | Sequence Number: 1. Shutdown Comment: \u7900\u7300"); + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + // 1 SER record (7 fields) + 3 OEM timestamped records (5 fields) = 4 total + Assert.AreEqual(4, monitor.DecodedSelCount); + } + } + + [Test] + public async Task IpmiUtilSelMonitorGenericDecodingCopiesDecodedFileToTimestampedPath() + { + this.SetupTest(); + + string decodedOutput = "0001 | [11/05/2025 18:56:57] | BMC, LUN 0 | Event Logging Disabled | ID:0xF1 | Log Area Reset/cleared | Asserted"; + + this.mockFixture.FileSystem.Setup(fs => fs.File.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.mockFixture.FileSystem.Setup(fs => fs.File.Exists(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")))) + .Returns(true); + this.mockFixture.FileSystem.Setup(fs => fs.File.ReadAllTextAsync(It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), It.IsAny())) + .ReturnsAsync(decodedOutput); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDirectory) => + { + return new InMemoryProcess(); + }; + + string selRecords = "cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b"; + + using (TestIpmiUtilSelMonitor monitor = new TestIpmiUtilSelMonitor(this.mockFixture)) + { + monitor.DecoderExePath = "crcsdkseldecoder.exe"; + + await monitor.TestExecuteGenericDecodingAsync(selRecords, EventContext.None, CancellationToken.None); + + Assert.AreEqual(1, monitor.DecodedSelCount); + } + + // Verify the decoded file was copied to a timestamped path + this.mockFixture.FileSystem.Verify( + fe => fe.File.Copy( + It.Is(s => s.Contains("All-BMC-SEL-Decoded.log")), + It.Is(s => s.Contains("_decoded_selRecords.log")), + true), + Times.Once); + } + + private void SetupGenericPackage() + { + var genericPackage = new DependencyPath("crcsdk_seldecoder", this.mockFixture.GetPackagePath("crcsdk_seldecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(genericPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(genericPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(genericPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(this.mockFixture.Combine(platformSpecificLocation, "crcsdkseldecoder.exe"))) + .Returns(true); + } + + private void SetupToolsetExistsInAlternateLocation(string expectedLocation) + { + // Setup: + // A toolset package containing ipmiutil does not exist on the system. However, the + // ipmiutil toolset DOES exist in one of the alternate locations. + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + InMemoryProcess process = new InMemoryProcess(); + + if (arguments.EndsWith("--help") || arguments.EndsWith("/?")) + { + // Setup: + // The toolset is not pre-installed on the system (e.g. via a Linux package manager). + process.ExitCode = 1; + } + + return process; + }; + + // Setup: + // The toolset is installed in an alternate location. + this.mockFixture.File.Setup(f => f.Exists(expectedLocation)).Returns(true); + } + + private void SetupToolsetExistsInPackage() + { + // Setup: + // The ipmiutil toolset package exists but not for the expected platform/architecture folder. + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(this.mockPackage.Path)) + .Returns(true); + + string toolset = this.mockFixture.Platform == PlatformID.Unix ? "ipmiutil" : "ipmiutil.exe"; + string platformSpecificToolsetLocation = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificToolsetLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(this.mockFixture.Combine(platformSpecificToolsetLocation, toolset))) + .Returns(true); + } + + private void SetupToolsetPreInstalled(int ipmiutilHelpExitCode) + { + // Setup: + // The ipmiutil toolset is pre-installed on the system. + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())).Returns(false); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Mock process = new Mock(); + + if (command.StartsWith("ipmiutil") && (arguments == "--help" || arguments == "/?")) + { + // Standard output is required to confirm the toolset is available. + process.Setup(command, arguments, workingDir, exitCode: ipmiutilHelpExitCode, standardOutput: "ipmiutil ver. 1.2.3"); + } + else + { + process.Setup(command, arguments, workingDir, exitCode: 0); + } + + return process.Object; + }; + } + + private void SetupSherlockPackage() + { + var sherlockPackage = new DependencyPath("sherlockSelDecoder", this.mockFixture.GetPackagePath("sherlockSelDecoder")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(sherlockPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(sherlockPackage.Path)) + .Returns(true); + + string platformSpecificLocation = this.mockFixture.Combine(sherlockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(this.mockFixture.Combine(platformSpecificLocation, "sel_decode.exe"))) + .Returns(true); + } + + public class TestSelRecord + { + public string RecordId { get; set; } + public string Timestamp { get; set; } + public string Severity { get; set; } + public string Source { get; set; } + public string EventType { get; set; } + public string Sensor { get; set; } + public string EventDetail { get; set; } + public bool IsHardwareError { get; set; } + } + + private class TestIpmiUtilSelMonitor : IpmiUtilSelMonitor + { + public TestIpmiUtilSelMonitor(MockFixture fixture) + : base(fixture.Dependencies, fixture.Parameters) + { + } + + public new string IpmiUtilExePath + { + get + { + return base.IpmiUtilExePath; + } + } + + public new Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeAsync(telemetryContext, cancellationToken); + } + + public new void CaptureMetrics(string results, string commandArguments, DateTime startTime, DateTime endTime, EventContext telemetryContext) + { + base.CaptureMetrics(results, commandArguments, startTime, endTime, telemetryContext); + } + + public TestSelRecord TestParseSelRecord(string line) + { + var record = base.ParseSelRecord(line); + if (record == null) return null; + + return new TestSelRecord + { + RecordId = record.RecordId, + Timestamp = record.Timestamp, + Severity = record.Severity, + Source = record.Source, + EventType = record.EventType, + Sensor = record.Sensor, + EventDetail = record.EventDetail, + IsHardwareError = record.IsHardwareError + }; + } + + public bool TestIsSelEventSuspected(TestSelRecord selRecord) + { + if (selRecord == null) return false; + + var record = new SelRecord + { + RecordId = selRecord.RecordId, + Timestamp = selRecord.Timestamp, + Severity = selRecord.Severity, + Source = selRecord.Source, + EventType = selRecord.EventType, + Sensor = selRecord.Sensor, + EventDetail = selRecord.EventDetail, + IsHardwareError = selRecord.IsHardwareError + }; + + return base.IsSelEventSuspected(record); + } + + public bool TestIsSelEventWarning(TestSelRecord selRecord) + { + if (selRecord == null) return false; + + var record = new SelRecord + { + RecordId = selRecord.RecordId, + Timestamp = selRecord.Timestamp, + Severity = selRecord.Severity, + Source = selRecord.Source, + EventType = selRecord.EventType, + Sensor = selRecord.Sensor, + EventDetail = selRecord.EventDetail, + IsHardwareError = selRecord.IsHardwareError + }; + + return base.IsSelEventWarning(record); + } + + public (IList suspectedEvents, IList warningEvents) TestAnalyzeSuspectedSelEvents(string results) + { + return base.AnalyzeSuspectedSelEvents(results); + } + + public int TestCountSelRecords(string results) + { + return base.CountSelRecords(results); + } + + public bool TestDetermineIfHardwareError(string severity, string eventDetail) + { + return base.DetermineIfHardwareError(severity, eventDetail); + } + + public Task TestCaptureSuspectedEventsAsync(string results, EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.CaptureSuspectedEventsAsync(results, telemetryContext, cancellationToken); + } + + public new string DecoderExePath + { + get + { + return base.DecoderExePath; + } + + set + { + base.DecoderExePath = value; + } + } + + public Task TestInitializeDecoderAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeDecoderAsync(telemetryContext, cancellationToken); + } + + public Task TestExecuteSherlockDecodingAsync(string selRecords, EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.ExecuteSherlockDecodingAsync(selRecords, telemetryContext, cancellationToken); + } + + public Task TestExecuteGenericDecodingAsync(string selRecords, EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.ExecuteGenericDecodingAsync(selRecords, telemetryContext, cancellationToken); + } + } + + private class TestIpmiUtilSelMonitor2 : IpmiUtilSelMonitor + { + public TestIpmiUtilSelMonitor2(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + public new string IpmiUtilExePath + { + get + { + return base.IpmiUtilExePath; + } + } + + public new Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeAsync(telemetryContext, cancellationToken); + } + + public new void CaptureMetrics(string results, string commandArguments, DateTime startTime, DateTime endTime, EventContext telemetryContext) + { + base.CaptureMetrics(results, commandArguments, startTime, endTime, telemetryContext); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMetricsParserTests.cs new file mode 100644 index 0000000000..0c4134d8ef --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMetricsParserTests.cs @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using NUnit.Framework; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class IpmiUtilSensorMetricsParserTests : MockFixture + { + private MockFixture mockFixture; + private string examplesDirectory; + + [SetUp] + public void SetupTest() + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + this.examplesDirectory = MockFixture.GetDirectory(typeof(IpmiUtilSensorMetricsParserTests), "test_examples", "ipmiutil"); + } + + [Test] + public void IpmiUtilSensorParserVerifyMetricsExample1() + { + string outputPath = this.mockFixture.Combine(this.examplesDirectory, "ipmiutil_sensor_example_1.txt"); + string text = System.IO.File.ReadAllText(outputPath); + IpmiUtilSensorMetricsParser testParser = new IpmiUtilSensorMetricsParser(); + IList metrics = testParser.Parse(text); + + Assert.AreEqual(103, metrics.Count); + MetricAssert.Exists(metrics, "CX7_0_Temp", 44, "celcius"); + MetricAssert.Exists(metrics, "CX7_1_Temp", 42, "celcius"); + MetricAssert.Exists(metrics, "CX7_2_Temp", 41, "celcius"); + MetricAssert.Exists(metrics, "CX7_3_Temp", 40, "celcius"); + MetricAssert.Exists(metrics, "E1S_1_TMP", 27, "celcius"); + MetricAssert.Exists(metrics, "E1S_2_TMP", 26, "celcius"); + MetricAssert.Exists(metrics, "E1S_3_TMP", 27, "celcius"); + MetricAssert.Exists(metrics, "E1S_4_TMP", 26, "celcius"); + MetricAssert.Exists(metrics, "E1S_5_TMP", 27, "celcius"); + MetricAssert.Exists(metrics, "E1S_TMP_MAX", 27, "celcius"); + MetricAssert.Exists(metrics, "FAN_0A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_0B", 20655, "rpm"); + MetricAssert.Exists(metrics, "FAN_1A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_1B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FAN_2A", 21756, "rpm"); + MetricAssert.Exists(metrics, "FAN_2B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FAN_3A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_3B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FAN_4A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_4B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FAN_5A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_5B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FAN_6A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_6B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FAN_7A", 21904, "rpm"); + MetricAssert.Exists(metrics, "FAN_7B", 20385, "rpm"); + MetricAssert.Exists(metrics, "FANS_SWB_PWM", 66, "percentage"); + MetricAssert.Exists(metrics, "GPU_DRAM_PWR_SUM", 100, "watts"); + MetricAssert.Exists(metrics, "GPU_DRAM_TMP_MAX", 37, "celcius"); + MetricAssert.Exists(metrics, "GPU_PWR_SUM", 880, "watts"); + MetricAssert.Exists(metrics, "GPU_TLIMIT_MIN", 55, "celcius"); + MetricAssert.Exists(metrics, "GPU0_DRAM0_PWR0", 25, "watts"); + MetricAssert.Exists(metrics, "GPU0_DRAM0_TMP0", 37, "celcius"); + MetricAssert.Exists(metrics, "GPU0_PWR0", 215, "watts"); + MetricAssert.Exists(metrics, "GPU0_TLIMIT_TMP1", 55, "celcius"); + MetricAssert.Exists(metrics, "GPU1_DRAM0_PWR0", 25, "watts"); + MetricAssert.Exists(metrics, "GPU1_DRAM0_TMP0", 35, "celcius"); + MetricAssert.Exists(metrics, "GPU1_PWR0", 240, "watts"); + MetricAssert.Exists(metrics, "GPU1_TLIMIT_TMP1", 55, "celcius"); + MetricAssert.Exists(metrics, "GPU2_DRAM0_PWR0", 20, "watts"); + MetricAssert.Exists(metrics, "GPU2_DRAM0_TMP0", 36, "celcius"); + MetricAssert.Exists(metrics, "GPU2_PWR0", 220, "watts"); + MetricAssert.Exists(metrics, "GPU2_TLIMIT_TMP1", 55, "celcius"); + MetricAssert.Exists(metrics, "GPU3_DRAM0_PWR0", 20, "watts"); + MetricAssert.Exists(metrics, "GPU3_DRAM0_TMP0", 35, "celcius"); + MetricAssert.Exists(metrics, "GPU3_PWR0", 210, "watts"); + MetricAssert.Exists(metrics, "GPU3_TLIMIT_TMP1", 56, "celcius"); + MetricAssert.Exists(metrics, "HMC_TMP", 35, "celcius"); + MetricAssert.Exists(metrics, "HSC_PWR_SUM", 1342, "watts"); + MetricAssert.Exists(metrics, "HSC0_INPUT_PWR", 115.36, "watts"); + MetricAssert.Exists(metrics, "HSC0_INPUT_VOLT", 51.03, "volts"); + MetricAssert.Exists(metrics, "HSC0_OUTPUT_CUR", 2.21, "amps"); + MetricAssert.Exists(metrics, "HSC0_TMP", 29, "celcius"); + MetricAssert.Exists(metrics, "HSC1_INPUT_PWR", 594, "watts"); + MetricAssert.Exists(metrics, "HSC1_INPUT_VOLT", 51.03, "volts"); + MetricAssert.Exists(metrics, "HSC1_OUTPUT_CUR", 12.35, "amps"); + MetricAssert.Exists(metrics, "HSC1_TMP", 24, "celcius"); + MetricAssert.Exists(metrics, "HSC2_INPUT_PWR", 594, "watts"); + MetricAssert.Exists(metrics, "HSC2_INPUT_VOLT", 51.34, "volts"); + MetricAssert.Exists(metrics, "HSC2_OUTPUT_CUR", 11.4, "amps"); + MetricAssert.Exists(metrics, "HSC2_TMP", 29, "celcius"); + MetricAssert.Exists(metrics, "LEAK_L_COLDPLATE", 1.74, "volts"); + MetricAssert.Exists(metrics, "LEAK_L_MANIFOLD", 1.74, "volts"); + MetricAssert.Exists(metrics, "LEAK_R_COLDPLATE", 1.74, "volts"); + MetricAssert.Exists(metrics, "LEAK_R_MANIFOLD", 1.73, "volts"); + MetricAssert.Exists(metrics, "M0_CPU_PWR0", 92, "watts"); + MetricAssert.Exists(metrics, "M0_CPU_TMPAVG0", 35, "celcius"); + MetricAssert.Exists(metrics, "M0_CPU_TMPLIMIT0", 52, "celcius"); + MetricAssert.Exists(metrics, "M0_VREG_CPUPWR0", 44, "watts"); + MetricAssert.Exists(metrics, "M0_VREG_CPUVOLT0", 0.8, "volts"); + MetricAssert.Exists(metrics, "M0_VREG_SOCPWR0", 6.16, "watts"); + MetricAssert.Exists(metrics, "M0_VREG_SOCVOLT0", 0.8, "volts"); + MetricAssert.Exists(metrics, "M1_CPU_PWR0", 90, "watts"); + MetricAssert.Exists(metrics, "M1_CPU_TMPAVG0", 35, "celcius"); + MetricAssert.Exists(metrics, "M1_CPU_TMPLIMIT0", 52, "celcius"); + MetricAssert.Exists(metrics, "M1_VREG_CPUPWR0", 42, "watts"); + MetricAssert.Exists(metrics, "M1_VREG_CPUVOLT0", 0.8, "volts"); + MetricAssert.Exists(metrics, "M1_VREG_SOCPWR0", 5.6, "watts"); + MetricAssert.Exists(metrics, "M1_VREG_SOCVOLT0", 0.8, "volts"); + MetricAssert.Exists(metrics, "M2_TMP", 30, "celcius"); + MetricAssert.Exists(metrics, "OL_S1_AMB_TMP", 38.5, "celcius"); + MetricAssert.Exists(metrics, "OL_S1_FPGA_TMP", 45, "celcius"); + MetricAssert.Exists(metrics, "OL_S1_HSC_TMP", 22.5, "celcius"); + MetricAssert.Exists(metrics, "OL_S1_POWER", 63.13, "watts"); + MetricAssert.Exists(metrics, "OL_S1_SOC_TMP", 48, "celcius"); + MetricAssert.Exists(metrics, "P12V_CB1_CUR", 33, "amps"); + MetricAssert.Exists(metrics, "P12V_CB1_TMP", 33, "celcius"); + MetricAssert.Exists(metrics, "P12V_CB1_VOLT", 11.96, "volts"); + MetricAssert.Exists(metrics, "P12V_CB2_CUR", 30, "amps"); + MetricAssert.Exists(metrics, "P12V_CB2_TMP", 36, "celcius"); + MetricAssert.Exists(metrics, "P12V_CB2_VOLT", 11.96, "volts"); + MetricAssert.Exists(metrics, "P12V_SCM_VOLT", 12.13, "volts"); + MetricAssert.Exists(metrics, "P12V_STBY_CUR", 6, "amps"); + MetricAssert.Exists(metrics, "P12V_STBY_TMP", 36, "celcius"); + MetricAssert.Exists(metrics, "P12V_STBY_VOLT", 11.96, "volts"); + MetricAssert.Exists(metrics, "P3V_BAT_VOLT", 3.18, "volts"); + MetricAssert.Exists(metrics, "P3V3_SCM_VOLT", 3.31, "volts"); + MetricAssert.Exists(metrics, "P5V_SCM_VOLT", 5.05, "volts"); + MetricAssert.Exists(metrics, "PCIESW_DIE_TEMP", 37, "celcius"); + MetricAssert.Exists(metrics, "PDB_OUTLET_TMP", 33.5, "celcius"); + MetricAssert.Exists(metrics, "SBP_INLET_TMP", 26.5, "celcius"); + MetricAssert.Exists(metrics, "SCM_INLET_TMP", 24, "celcius"); + MetricAssert.Exists(metrics, "SWB_INLET_TMP", 27.5, "celcius"); + } + + [Test] + public void IpmiUtilSensorParserVerifyMetricsExample2() + { + string outputPath = this.mockFixture.Combine(this.examplesDirectory, "ipmiutil_sensor_example_2.txt"); + string text = System.IO.File.ReadAllText(outputPath); + IpmiUtilSensorMetricsParser testParser = new IpmiUtilSensorMetricsParser(); + IList metrics = testParser.Parse(text); + + Assert.AreEqual(122, metrics.Count); // there are 122 SDRType = Full records in this example + + // Verify all 122 metrics + var okTag = new List() { "OK" }; + var initTag = new List() { "Init" }; + + MetricAssert.Exists(metrics, "FANS_SWB_PWM", 26, "percentage", okTag); + MetricAssert.Exists(metrics, "P12V_SCM_VOLT", 12.15, "volts", okTag); + MetricAssert.Exists(metrics, "P5V_SCM_VOLT", 5.07, "volts", okTag); + MetricAssert.Exists(metrics, "P3V3_SCM_VOLT", 3.32, "volts", okTag); + MetricAssert.Exists(metrics, "P3V_BAT_VOLT", 3.34, "volts", okTag); + MetricAssert.Exists(metrics, "SCM_INLET_TMP", 26.5, "celcius", okTag); + MetricAssert.Exists(metrics, "SWB_INLET_TMP", 31, "celcius", okTag); + MetricAssert.Exists(metrics, "SBP_INLET_TMP", 31, "celcius", okTag); + MetricAssert.Exists(metrics, "M2_TMP", 42, "celcius", okTag); + MetricAssert.Exists(metrics, "E1S_1_TMP", 33, "celcius", okTag); + MetricAssert.Exists(metrics, "E1S_2_TMP", 32, "celcius", okTag); + MetricAssert.Exists(metrics, "E1S_3_TMP", 33, "celcius", okTag); + MetricAssert.Exists(metrics, "E1S_4_TMP", 33, "celcius", okTag); + MetricAssert.Exists(metrics, "E1S_5_TMP", 32, "celcius", okTag); + MetricAssert.Exists(metrics, "E1S_TMP_MAX", 33, "celcius", okTag); + MetricAssert.Exists(metrics, "PDB_OUTLET_TMP", 42, "celcius", okTag); + MetricAssert.Exists(metrics, "HSC0_INPUT_PWR", 120, "watts", okTag); + MetricAssert.Exists(metrics, "HSC0_INPUT_VOLT", 51.34, "volts", okTag); + MetricAssert.Exists(metrics, "HSC0_OUTPUT_CUR", 2.2, "amps", okTag); + MetricAssert.Exists(metrics, "HSC0_TMP", 37, "celcius", okTag); + MetricAssert.Exists(metrics, "HSC1_INPUT_PWR", 550, "watts", okTag); + MetricAssert.Exists(metrics, "HSC1_INPUT_VOLT", 51.34, "volts", okTag); + MetricAssert.Exists(metrics, "HSC1_OUTPUT_CUR", 11, "amps", okTag); + MetricAssert.Exists(metrics, "HSC1_TMP", 33, "celcius", okTag); + MetricAssert.Exists(metrics, "HSC2_INPUT_PWR", 550, "watts", okTag); + MetricAssert.Exists(metrics, "HSC2_INPUT_VOLT", 51.03, "volts", okTag); + MetricAssert.Exists(metrics, "HSC2_OUTPUT_CUR", 9.9, "amps", okTag); + MetricAssert.Exists(metrics, "HSC2_TMP", 38, "celcius", okTag); + MetricAssert.Exists(metrics, "HSC_PWR_SUM", 1236, "watts", okTag); + MetricAssert.Exists(metrics, "P12V_STBY_VOLT", 11.96, "volts", okTag); + MetricAssert.Exists(metrics, "P12V_STBY_CUR", 8.5, "amps", okTag); + MetricAssert.Exists(metrics, "P12V_STBY_TMP", 46, "celcius", okTag); + MetricAssert.Exists(metrics, "P12V_CB1_VOLT", 12.51, "volts", okTag); + MetricAssert.Exists(metrics, "P12V_CB1_CUR", 45, "amps", okTag); + MetricAssert.Exists(metrics, "P12V_CB1_TMP", 43, "celcius", okTag); + MetricAssert.Exists(metrics, "P12V_CB2_VOLT", 12.51, "volts", okTag); + MetricAssert.Exists(metrics, "P12V_CB2_CUR", 37.5, "amps", okTag); + MetricAssert.Exists(metrics, "P12V_CB2_TMP", 46, "celcius", okTag); + MetricAssert.Exists(metrics, "FAN_0A", 8436, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_0B", 7290, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_1A", 8288, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_1B", 7290, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_2A", 8288, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_2B", 7155, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_3A", 8288, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_3B", 7155, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_4A", 8140, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_4B", 7290, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_5A", 8140, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_5B", 7155, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_6A", 8288, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_6B", 7155, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_7A", 8288, "rpm", okTag); + MetricAssert.Exists(metrics, "FAN_7B", 7155, "rpm", okTag); + MetricAssert.Exists(metrics, "GPU0_DRAM0_TMP0", 37, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU1_DRAM0_TMP0", 37, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU2_DRAM0_TMP0", 36, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU3_DRAM0_TMP0", 36, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU_DRAM_TMP_MAX", 37, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU0_PWR0", 180, "watts", okTag); + MetricAssert.Exists(metrics, "GPU1_PWR0", 185, "watts", okTag); + MetricAssert.Exists(metrics, "GPU2_PWR0", 190, "watts", okTag); + MetricAssert.Exists(metrics, "GPU3_PWR0", 185, "watts", okTag); + MetricAssert.Exists(metrics, "GPU_PWR_SUM", 740, "watts", okTag); + MetricAssert.Exists(metrics, "GPU0_TLIMIT_TMP1", 50, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU1_TLIMIT_TMP1", 50, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU2_TLIMIT_TMP1", 50, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU3_TLIMIT_TMP1", 51, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU_TLIMIT_MIN", 50, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU0_DRAM0_PWR0", 35, "watts", okTag); + MetricAssert.Exists(metrics, "GPU1_DRAM0_PWR0", 35, "watts", okTag); + MetricAssert.Exists(metrics, "GPU2_DRAM0_PWR0", 35, "watts", okTag); + MetricAssert.Exists(metrics, "GPU3_DRAM0_PWR0", 30, "watts", okTag); + MetricAssert.Exists(metrics, "GPU_DRAM_PWR_SUM", 140, "watts", okTag); + MetricAssert.Exists(metrics, "M0_CPU_PWR0", 98, "watts", okTag); + MetricAssert.Exists(metrics, "M0_VREG_CPUPWR0", 46.8, "watts", okTag); + MetricAssert.Exists(metrics, "M0_VREG_SOCPWR0", 6.6, "watts", okTag); + MetricAssert.Exists(metrics, "M0_CPU_TMPAVG0", 41, "celcius", okTag); + MetricAssert.Exists(metrics, "M0_CPU_TMPLIMIT0", 47, "celcius", okTag); + MetricAssert.Exists(metrics, "M0_VREG_CPUVOLT0", 0.8, "volts", okTag); + MetricAssert.Exists(metrics, "M0_VREG_SOCVOLT0", 0.8, "volts", okTag); + MetricAssert.Exists(metrics, "M1_CPU_PWR0", 96, "watts", okTag); + MetricAssert.Exists(metrics, "M1_VREG_CPUPWR0", 45.6, "watts", okTag); + MetricAssert.Exists(metrics, "M1_VREG_SOCPWR0", 6, "watts", okTag); + MetricAssert.Exists(metrics, "M1_CPU_TMPAVG0", 41, "celcius", okTag); + MetricAssert.Exists(metrics, "M1_CPU_TMPLIMIT0", 47, "celcius", okTag); + MetricAssert.Exists(metrics, "M1_VREG_CPUVOLT0", 0.8, "volts", okTag); + MetricAssert.Exists(metrics, "M1_VREG_SOCVOLT0", 0.8, "volts", okTag); + MetricAssert.Exists(metrics, "OL_S1_HSC_TMP", 32, "celcius", okTag); + MetricAssert.Exists(metrics, "OL_S1_FPGA_TMP", 76, "celcius", okTag); + MetricAssert.Exists(metrics, "OL_S1_AMB_TMP", 64, "celcius", okTag); + MetricAssert.Exists(metrics, "OL_S1_SOC_TMP", 88, "celcius", okTag); + MetricAssert.Exists(metrics, "OL_S1_POWER", 70.8, "watts", okTag); + MetricAssert.Exists(metrics, "PCIESW_DIE_TEMP", 44, "celcius", okTag); + MetricAssert.Exists(metrics, "HMC_TMP", 39, "celcius", okTag); + MetricAssert.Exists(metrics, "LEAK_L_COLDPLATE", 1.72, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_L_MANIFOLD", 1.73, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_R_COLDPLATE", 1.73, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_R_MANIFOLD", 1.7, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_L_TRAY", 2.27, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_R_TRAY", 2.23, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_Reserved_0", 3.29, "volts", okTag); + MetricAssert.Exists(metrics, "LEAK_Reserved_1", 3.26, "volts", okTag); + MetricAssert.Exists(metrics, "CX8_0_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_1_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_2_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_3_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_0_OSFP_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_1_OSFP_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_2_OSFP_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "CX8_3_OSFP_Temp", 0, "celcius", initTag); + MetricAssert.Exists(metrics, "GPU0_AVG_TMP", 36, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU1_AVG_TMP", 36, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU2_AVG_TMP", 36, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU3_AVG_TMP", 35, "celcius", okTag); + MetricAssert.Exists(metrics, "GPU_AVG_TMP_MAX", 36, "celcius", okTag); + MetricAssert.Exists(metrics, "M0_inlet_Temp0", 38, "celcius", okTag); + MetricAssert.Exists(metrics, "M0_inlet_Temp1", 46, "celcius", okTag); + MetricAssert.Exists(metrics, "M0_Exhaust_Temp0", 45, "celcius", okTag); + MetricAssert.Exists(metrics, "M1_inlet_Temp0", 43, "celcius", okTag); + MetricAssert.Exists(metrics, "M1_inlet_Temp1", 47, "celcius", okTag); + MetricAssert.Exists(metrics, "M1_Exhaust_Temp0", 43, "celcius", okTag); + + + var descriptions = metrics.Select(x => x.Description); + Assert.AreEqual(122, descriptions.Distinct().Count(), "All metrics should have unique descriptions"); + + Assert.IsTrue(descriptions.Contains("003a | Full | Fan | 3a | FAN_0A | OK | 8436.00 RPM")); + Assert.IsTrue(descriptions.Contains("0001 | Full | Fan | 01 | FANS_SWB_PWM | OK | 26.00 %")); + Assert.IsTrue(descriptions.Contains("005a | Full | Temperature | 5a | GPU0_DRAM0_TMP0 | OK | 37.00 C")); + Assert.IsTrue(descriptions.Contains("001e | Full | Temperature | 1e | PDB_OUTLET_TMP | OK | 42.00 C")); + + foreach (Metric metric in metrics) + { + var arr = metric.Description.Split('|'); + + Assert.AreEqual(metric.Metadata["id"], arr[0].Trim()); + Assert.AreEqual(metric.Metadata["sdrType"], arr[1].Trim()); + Assert.AreEqual(metric.Metadata["type"], arr[2].Trim()); + Assert.AreEqual(metric.Metadata["snum"], arr[3].Trim()); + Assert.AreEqual(metric.Metadata["name"], arr[4].Trim()); + Assert.AreEqual(metric.Metadata["status"], arr[5].Trim()); + } + } + + [Test] + public void IpmiUtilSensorParserVerifyInvalidEntriesExample3() + { + string outputPath = this.mockFixture.Combine(this.examplesDirectory, "ipmiutil_sensor_example_3.txt"); + string text = System.IO.File.ReadAllText(outputPath); + IpmiUtilSensorMetricsParser testParser = new IpmiUtilSensorMetricsParser(); + IList metrics = testParser.Parse(text); + + // Verify that metrics with valid numerical readings were parsed correctly + Assert.IsNotNull(metrics); + Assert.Greater(metrics.Count, 0); + + // Verify specific metrics that should have been parsed successfully + MetricAssert.Exists(metrics, "P12V_SCM_VOLT", 12.13, "volts"); + MetricAssert.Exists(metrics, "P5V_SCM_VOLT", 5.05, "volts"); + MetricAssert.Exists(metrics, "P3V_BAT_VOLT", 3.18, "volts"); + MetricAssert.Exists(metrics, "M2_TMP", 30, "celcius"); + MetricAssert.Exists(metrics, "HSC0_TMP", 29, "celcius"); + + // Verify that invalid sensor entries were captured (header line is now skipped) + Assert.IsNotNull(testParser.InvalidIpmiSensorEntries); + Assert.AreEqual(17, testParser.InvalidIpmiSensorEntries.Count); + + // Verify specific invalid sensor entries with different malformed patterns + var fansPwmEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "FANS_SWB_PWM"); + Assert.IsNotNull(fansPwmEvent); + Assert.AreEqual("0001", fansPwmEvent["id"]); + Assert.AreEqual("Fan", fansPwmEvent["type"]); + Assert.AreEqual("NA", fansPwmEvent["reading"]); + + var p3v3ScmVoltEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "P3V3_SCM_VOLT"); + Assert.IsNotNull(p3v3ScmVoltEvent); + Assert.AreEqual("0011", p3v3ScmVoltEvent["id"]); + Assert.AreEqual("Voltage", p3v3ScmVoltEvent["type"]); + Assert.AreEqual("...", p3v3ScmVoltEvent["reading"]); + + var sbpInletTmpEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "SBP_INLET_TMP"); + Assert.IsNotNull(sbpInletTmpEvent); + Assert.AreEqual("0016", sbpInletTmpEvent["id"]); + Assert.AreEqual("Temperature", sbpInletTmpEvent["type"]); + Assert.AreEqual("..", sbpInletTmpEvent["reading"]); + + var e1s1TmpEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "E1S_1_TMP"); + Assert.IsNotNull(e1s1TmpEvent); + Assert.AreEqual("0018", e1s1TmpEvent["id"]); + Assert.AreEqual("Na", e1s1TmpEvent["reading"]); + + var hsc0InputPwrEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "HSC0_INPUT_PWR"); + Assert.IsNotNull(hsc0InputPwrEvent); + Assert.AreEqual("001f", hsc0InputPwrEvent["id"]); + Assert.AreEqual("-", hsc0InputPwrEvent["reading"]); + + var hsc0OutputCurEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "HSC0_OUTPUT_CUR"); + Assert.IsNotNull(hsc0OutputCurEvent); + Assert.AreEqual("0021", hsc0OutputCurEvent["id"]); + Assert.AreEqual("N/A", hsc0OutputCurEvent["reading"]); + + var hsc1TmpEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "HSC1_TMP"); + Assert.IsNotNull(hsc1TmpEvent); + Assert.AreEqual("0026", hsc1TmpEvent["id"]); + Assert.AreEqual("ERR", hsc1TmpEvent["reading"]); + + var fan6aEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "FAN_6A"); + Assert.IsNotNull(fan6aEvent); + Assert.AreEqual("0046", fan6aEvent["id"]); + Assert.AreEqual("UNKNOWN", fan6aEvent["reading"]); + + var invalidVoltEvent = testParser.InvalidIpmiSensorEntries.FirstOrDefault(e => e["name"]?.ToString() == "M1_VREG_CPUVOLT0"); + Assert.IsNotNull(invalidVoltEvent); + Assert.AreEqual("008e", invalidVoltEvent["id"]); + Assert.AreEqual("INVALID", invalidVoltEvent["reading"]); + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMonitorTests.cs b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMonitorTests.cs new file mode 100644 index 0000000000..9919767aed --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSensorMonitorTests.cs @@ -0,0 +1,285 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Runtime.InteropServices; + using System.Threading; + using System.Threading.Tasks; + using Moq; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class IpmiUtilSensorMonitorTests : MockFixture + { + private MockFixture mockFixture; + private DependencyPath mockPackage; + + public void SetupTest(PlatformID platform = PlatformID.Win32NT, Architecture architecture = Architecture.X64) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(platform, architecture); + this.mockFixture.Parameters["PackageName"] = "ipmiutil"; + + // Setup: + // The monitor will get the ipmiutil toolset from a package downloaded by default. + this.mockPackage = new DependencyPath("ipmiutil", this.mockFixture.GetPackagePath("ipmiutil")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + + this.mockFixture.File.Reset(); + this.mockFixture.Directory.Reset(); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public async Task IpmiUtilSensorMonitorChecksForTheToolsetInAPackageLocation(PlatformID expectedPlatform, Architecture expectedArchitecture) + { + this.SetupTest(expectedPlatform, expectedArchitecture); + + // Setup: + // The ipmiutil toolset package exists but not for the expected platform/architecture folder. + this.SetupToolsetExistsInPackage(); + + using (TestIpmiUtilSensorMonitor monitor = new TestIpmiUtilSensorMonitor(this.mockFixture)) + { + await monitor.InitializeAsync(EventContext.None, CancellationToken.None); + + string expectedToolset = expectedPlatform == PlatformID.Unix ? "ipmiutil" : "ipmiutil.exe"; + + // Expectation: + // The path should be to the location in the dependency package. + string expectedLocation = this.mockFixture.Combine( + this.mockPackage.Path, + this.mockFixture.PlatformArchitectureName, + expectedToolset); + + string actualLocation = monitor.IpmiUtilExePath; + + Assert.AreEqual(expectedLocation, actualLocation); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void IpmiUtilSensorMonitorHandlesScenariosWhereToolsetPackagesExistButNotForTheParticularPlatformArchitecture(PlatformID expectedPlatform, Architecture expectedArchitecture) + { + this.SetupTest(expectedPlatform, expectedArchitecture); + + // Setup: + // The ipmiutil toolset package exists... + this.SetupToolsetExistsInPackage(); + + // ...but not for the expected platform/architecture folder. + string platformSpecificToolsetLocation = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificToolsetLocation)) + .Returns(false); + + using (TestIpmiUtilSensorMonitor monitor = new TestIpmiUtilSensorMonitor(this.mockFixture)) + { + // Expectation: + // An exception should not surface nor should the path have been set to a platform/specific + // path location (as it does not exist). + Assert.DoesNotThrowAsync(() => monitor.InitializeAsync(EventContext.None, CancellationToken.None)); + Assert.IsNull(monitor.IpmiUtilExePath); + } + } + + [Test] + [TestCase(0)] + [TestCase(234)] + public async Task IpmiUtilSensorMonitorChecksForTheToolsetInACommonLocationDefinedByThePathEnvironmentVariable(int successExitCode) + { + this.SetupTest(); + + // Setup: + // A toolset package containing ipmiutil does not exist on the system. Additionally, the + // ipmiutil toolset does not exist in one of the alternate locations. It does however exist + // in a common location as defined by the PATH environment variable. + this.SetupToolsetPreInstalled(successExitCode); + + using (TestIpmiUtilSensorMonitor monitor = new TestIpmiUtilSensorMonitor(this.mockFixture)) + { + await monitor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Expectation: + // The path should be to the alternate location. + string expectedLocation = "ipmiutil.exe"; + string actualLocation = monitor.IpmiUtilExePath; + + Assert.AreEqual(expectedLocation, actualLocation); + } + } + + [Test] + [TestCase(PlatformID.Win32NT, "C:\\BladeFX_latest\\BladeFX\\Tools\\IpmiUtil\\ipmiutil.exe")] + public async Task IpmiUtilSensorMonitorChecksForTheToolsetInAlternatePathLocations(PlatformID platform, string expectedLocation) + { + this.SetupTest(platform); + + // Setup: + // A toolset package containing ipmiutil does not exist on the system. However, the + // ipmiutil toolset DOES exist in one of the alternate locations. + this.SetupToolsetExistsInAlternateLocation(expectedLocation); + + using (TestIpmiUtilSensorMonitor monitor = new TestIpmiUtilSensorMonitor(this.mockFixture)) + { + await monitor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Expectation: + // The path should be to the alternate location. + string actualLocation = monitor.IpmiUtilExePath; + + Assert.AreEqual(expectedLocation, actualLocation); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void IpmiUtilSensorMonitorExitsGracefullyIfTheToolsetCannotBeLocated(PlatformID expectedPlatform, Architecture expectedArchitecture) + { + this.SetupTest(expectedPlatform, expectedArchitecture); + + // Setup: + // A toolset package containing ipmiutil does not exist on the system. Additionally, the + // ipmiutil toolset does not exist in one of the alternate locations. And finally, it does + // not exist in a common location as defined by the PATH environment variable. + // + // ...nowhere to be found!! + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Mock process = new Mock(); + + if ($"{command} {arguments}" == "ipmiutil --help" || $"{command} {arguments}" == "ipmiutil.exe /?") + { + process.Setup(command, arguments, workingDir, exitCode: 1); + } + else + { + Assert.Fail("Code flow should not proceed if the ipmiutil toolset path cannot be located."); + } + + return process.Object; + }; + + using (TestIpmiUtilSensorMonitor monitor = new TestIpmiUtilSensorMonitor(this.mockFixture)) + { + // Expectation: + // The monitor should exit gracefully and there should not be any path location + // identified for the ipmiutil toolset. + + Assert.DoesNotThrowAsync(() => monitor.ExecuteAsync(CancellationToken.None)); + Assert.IsNull(monitor.IpmiUtilExePath); + } + } + + private void SetupToolsetExistsInAlternateLocation(string expectedLocation) + { + // Setup: + // A toolset package containing ipmiutil does not exist on the system. However, the + // ipmiutil toolset DOES exist in one of the alternate locations. + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + InMemoryProcess process = new InMemoryProcess(); + + if (arguments.EndsWith("--help") || arguments.EndsWith("/?")) + { + // Setup: + // The toolset is not pre-installed on the system (e.g. via a Linux package manager). + process.ExitCode = 1; + } + + return process; + }; + + // Setup: + // The toolset is installed in an alternate location. + this.mockFixture.File.Setup(f => f.Exists(expectedLocation)).Returns(true); + } + + private void SetupToolsetExistsInPackage() + { + // Setup: + // The ipmiutil toolset package exists but not for the expected platform/architecture folder. + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPackage); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(this.mockPackage.Path)) + .Returns(true); + + string toolset = this.mockFixture.Platform == PlatformID.Unix ? "ipmiutil" : "ipmiutil.exe"; + string platformSpecificToolsetLocation = this.mockFixture.Combine(this.mockPackage.Path, this.mockFixture.PlatformArchitectureName); + + this.mockFixture.Directory + .Setup(dir => dir.Exists(platformSpecificToolsetLocation)) + .Returns(true); + + this.mockFixture.File + .Setup(file => file.Exists(this.mockFixture.Combine(platformSpecificToolsetLocation, toolset))) + .Returns(true); + } + + private void SetupToolsetPreInstalled(int ipmiutilHelpExitCode) + { + // Setup: + // The ipmiutil toolset is pre-installed on the system. + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(null as DependencyPath); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())).Returns(false); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Mock process = new Mock(); + + if (command.StartsWith("ipmiutil") && (arguments == "--help" || arguments == "/?")) + { + // Standard output is required to confirm the toolset is available. + process.Setup(command, arguments, workingDir, exitCode: ipmiutilHelpExitCode, standardOutput: "ipmiutil ver. 1.2.3"); + } + else + { + process.Setup(command, arguments, workingDir, exitCode: 0); + } + + return process.Object; + }; + } + + private class TestIpmiUtilSensorMonitor : IpmiUtilSensorMonitor + { + public TestIpmiUtilSensorMonitor(MockFixture fixture) + : base(fixture.Dependencies, fixture.Parameters) + { + } + + public new string IpmiUtilExePath + { + get + { + return base.IpmiUtilExePath; + } + } + + + public new Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeAsync(telemetryContext, cancellationToken); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Monitors/AssemblyInfo.cs b/src/VirtualClient/VirtualClient.Monitors/AssemblyInfo.cs index 3f83098656..4cc30baf12 100644 --- a/src/VirtualClient/VirtualClient.Monitors/AssemblyInfo.cs +++ b/src/VirtualClient/VirtualClient.Monitors/AssemblyInfo.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; using VirtualClient.Contracts; [assembly: VirtualClientComponentAssembly] +[assembly: InternalsVisibleTo("VirtualClient.Monitors.UnitTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100ff6dccd60f1b1d8a4c0cbb5481e9aee8d193b61c5402b0ffd28f015d22402539894d63eee174845d816fe36b1baba6c6724a19fd62b27d12576327a1d6ba29ed7e1a0021dbdf53c5367536f9f40190060569cee75378b8196c6001773a330e36ce0d50814338c0b064289749ccb58dc292d7440791a2760bde7dbfe894860bc8")] diff --git a/src/VirtualClient/VirtualClient.Monitors/ETW/WindowsETWMonitor.cs b/src/VirtualClient/VirtualClient.Monitors/ETW/WindowsETWMonitor.cs new file mode 100644 index 0000000000..59546df89e --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors/ETW/WindowsETWMonitor.cs @@ -0,0 +1,748 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Diagnostics.Tracing; + using Microsoft.Diagnostics.Tracing.Parsers; + using Microsoft.Diagnostics.Tracing.Session; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// Provides features for capturing the Windows ETW events + /// of interest. + /// + [SupportedPlatforms("win-x64,win-arm64", throwError: false)] + public class WindowsETWMonitor : VirtualClientComponent + { + private readonly object lockObject = new object(); + private readonly object lockProvider = new object(); + private TraceEventSession session; + private bool disposed; + private BlockingCollection queue = new BlockingCollection(); + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// + /// Parameters defined in the execution profile or supplied to the Virtual Client on the command line. + /// + public WindowsETWMonitor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + if (this.ProfilingMode == ProfilingMode.OnDemand) + { + VirtualClientRuntime.SendReceiveInstructions += this.OnProfileSystemOnDemand; + } + } + + /// + /// Stores the emitted event data + /// + protected Dictionary ETWData { get; set; } + + /// + /// Filters applied on the emitted events + /// + protected Dictionary> Filters { get; set; } + + /// + /// To capture the event data for Kernel/NonKernel provider type + /// + private string ProviderType + { + get + { + return this.Parameters.GetValue(nameof(WindowsETWMonitor.ProviderType), string.Empty); + } + } + + /// + /// To capture the event data for provider defined + /// + private dynamic Provider + { + get + { + string provider = this.Parameters.GetValue(nameof(WindowsETWMonitor.Provider), string.Empty); + return Guid.TryParse(provider, out Guid providerName) ? + providerName : provider == string.Empty ? null : provider; + } + } + + /// + /// To enable the Kernel provider keywords defined + /// + private List ProviderKeywords + { + get + { + this.Parameters.TryGetCollection(nameof(WindowsETWMonitor.ProviderKeywords), out IEnumerable keywords); + return keywords?.ToList(); + } + } + + /// + /// To capture the specific kernel events defined + /// + private List ProviderEvents + { + get + { + this.Parameters.TryGetCollection(nameof(WindowsETWMonitor.ProviderEvents), out IEnumerable events); + return events?.ToList(); + } + } + + /// + /// Filters to be applied on the captured events + /// + private string JsonFilters + { + get + { + return this.Parameters.GetValue(nameof(WindowsETWMonitor.JsonFilters), string.Empty); + } + } + + /// + /// Disposes of resources used by the instance. + /// + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (disposing) + { + if (!this.disposed) + { + if (this.ProfilingMode == ProfilingMode.OnDemand) + { + VirtualClientRuntime.SendReceiveInstructions -= this.OnProfileSystemOnDemand; + } + + if (this.session != null) + { + this.session.Dispose(); + } + + this.queue.Dispose(); + this.disposed = true; + } + } + } + + /// + /// Sets up this monitor + /// + protected override Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (this.Platform == PlatformID.Win32NT) + { + if ((Environment.OSVersion.Version.Major * 10) + (Environment.OSVersion.Version.Minor) < 62) + { + throw new MonitorException("This monitor works on Win8/Win 2012 an above only.", ErrorReason.PlatformNotSupported); + } + + if (TraceEventSession.IsElevated() != true) + { + throw new MonitorException("The monitor application must run with elevated privileges (Admin).", ErrorReason.Unauthorized); + } + } + else + { + throw new MonitorException("This monitor works on Win8/Win 2012 an above only.", ErrorReason.PlatformNotSupported); + } + + if (!string.IsNullOrEmpty(this.JsonFilters)) + { + try + { + JToken.Parse(this.JsonFilters); + this.Filters = JsonConvert.DeserializeObject>>(this.JsonFilters.ToLower()); + } + catch + { + throw new MonitorException( + $"Unexpected profile definition. One or more of the parameters in the profile" + + $"'{nameof(WindowsETWMonitor.JsonFilters)}' is not in the correct format. It require the parameter" + + $"to be defined in the JSON format and value of key value pair should be in an array." + + $"Ex:- \"{nameof(WindowsETWMonitor.JsonFilters)}\" : \"{{'EventName':['ProcessStart'],'Level':['2','4']}}\"", + ErrorReason.InvalidProfileDefinition); + } + } + + return Task.CompletedTask; + } + + /// + /// validate the parameters + /// + protected override void Validate() + { + if (string.IsNullOrEmpty(this.ProviderType)) + { + throw new MonitorException( + $"Unexpected profile definition. One or more of the parameters in the profile does not contain the " + + $"required '{nameof(WindowsETWMonitor.ProviderType)}' arguments defined. It require the provider type" + + $"to be defined to enable Kernel/NonKernel provider.", + ErrorReason.InvalidProfileDefinition); + } + + if (this.Provider == null) + { + throw new MonitorException( + $"Unexpected profile definition. One or more of the parameters in the profile does not contain the " + + $"required '{nameof(WindowsETWMonitor.Provider)}' arguments defined. It require the provider " + + $"to be defined to capture the events of the provider.", + ErrorReason.InvalidProfileDefinition); + } + + if (this.ProviderType == "Kernel" && this.ProviderKeywords == null) + { + throw new MonitorException( + $"Unexpected profile definition. One or more of the parameters in the profile does not contain the " + + $"required '{nameof(WindowsETWMonitor.ProviderKeywords)}' arguments defined. It require the keywords" + + $"to be defined to enable Kernel Provider." + + $"Ex:- \"{nameof(WindowsETWMonitor.ProviderKeywords)}\" : \"ImageLoad,Process\" ", + ErrorReason.InvalidProfileDefinition); + } + } + + /// + /// Executes the background monitoring process.. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + // All background monitors should return a background Task immediately to avoid blocking + // the thread. + + return Task.Run(async () => + { + try + { + if (this.ProfilingEnabled && this.ProfilingMode != ProfilingMode.None) + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Profiling Mode = '{this.ProfilingMode}'"); + await this.ExecuteWorkloadAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Expected when a Task.Delay is cancelled. + } + }); + } + + /// + /// Helper function that executes background monitoring process (see ExecuteAsync). + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (this.ProfilingMode == ProfilingMode.OnDemand) + { + await this.ExecuteProfilingOnDemandAsync(cancellationToken) + .ConfigureAwait(false); + } + else if (this.ProfilingMode == ProfilingMode.Interval) + { + await this.ExecuteProfilingOnIntervalAsync(cancellationToken) + .ConfigureAwait(false); + } + else + { + throw new NotSupportedException( + $"Profiling mode '{this.ProfilingMode}' not supported. Valid modes supported are '{ProfilingMode.Interval}' and '{ProfilingMode.OnDemand}'"); + } + } + + private async Task ExecuteProfilingOnDemandAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + // The profiler will cycle around doing nothing when it is in on-demand mode. + // It is waiting for notifications to profile during this time and will take action + // when it receives those notifications. + await Task.Delay(500, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected when a Task.Delay is cancelled. + } + catch + { + // Continue to attempt running the profiler. + } + } + } + + private void OnProfileSystemOnDemand(object sender, InstructionsEventArgs args) + { + lock (this.lockObject) + { + try + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] On-Demand Profiling Begin"); + + ProfilerInstructions instructions = (args.Instructions as ProfilerInstructions); + + // We only care about requests/instructions to profile the system. + if (instructions != null && instructions.ProfilingEnabled) + { + // Allow for the warm-up period. + Task.Delay(instructions.ProfilingWarmUpPeriod).GetAwaiter().GetResult(); + + if (!args.CancellationToken.IsCancellationRequested) + { + // Task will log important events as they are captured and queued. + this.ExecuteOnDemandMonitorProfileAsync(args.CancellationToken).GetAwaiter().GetResult(); + } + } + } + catch (OperationCanceledException) + { + // Expected when a Task.Delay is cancelled. + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + finally + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] On-Demand Profiling End"); + } + } + } + + /// + /// Starts the ETW process recording. + /// + private Task ExecuteOnDemandMonitorProfileAsync(CancellationToken cancellationToken) + { + return Task.Run(async () => + { + this.session = null; + try + { + // creating a unique session name to run simultaneously. + using (this.session = new TraceEventSession($"{this.Provider.Replace(" ", string.Empty)}_{Guid.NewGuid().ToString()}", null)) + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Started OnDemand Trace Event Session : {this.session.SessionName}"); + + // Enable providers + await this.StartETWTrace(this.ProviderType, this.Provider, this.ProviderKeywords, this.ProviderEvents, this.Filters).ConfigureAwait(false); + + // This is a blocking call. It will end when session is disposed. + Task recordEventsTask = Task.Run(() => this.session.Source.Process()); + + // Run the ETW monitoring for monitoringTimeSec and stop. + while (!cancellationToken.IsCancellationRequested) + { + IDictionary data = null; + try + { + // Take the item from queue and log to the system events + if (this.queue.TryTake(out data, 0, cancellationToken)) + { + IDictionary eventInfo = new Dictionary(); + foreach (DictionaryEntry entry in data) + { + eventInfo[entry.Key.ToString()] = entry.Value; + } + + string eventId = null; + if (data.Contains("EventID")) + { + eventId = data["EventID"].ToString(); + } + + string eventDescription = null; + if (data.Contains("EventName")) + { + eventDescription = data["EventName"].ToString(); + } + + this.Logger.LogSystemEvent( + "ETW", + "Windows ETW", + eventId, + LogLevel.Information, + EventContext.Persisted(), + null, + eventDescription, + eventInfo); + } + } + catch (InvalidOperationException) + { + // An InvalidOperationException means that Take() was called on a completed collection + } + catch (OperationCanceledException) + { + // Doesn't allow any more items to be added + this.queue.CompleteAdding(); + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + + await Task.Delay(500, cancellationToken).ConfigureAwait(false); + } + } + } + catch (OperationCanceledException) + { + // Expected when a Task.Delay is cancelled. + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + finally + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Completed On Demand Trace Event Session {this.session.SessionName}"); + // Stop the session after completing + this.session?.Stop(); + + // Close the session and clean up any resources associated with the session. + // Itis OK to call this more than once. + if (this.session != null) + { + this.session.Dispose(); + } + } + }, cancellationToken); + } + + private async Task ExecuteProfilingOnIntervalAsync(CancellationToken cancellationToken) + { + await Task.Delay(this.ProfilingWarmUpPeriod, cancellationToken).ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + try + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Interval Profiling Begin"); + + await this.ExecuteIntervalMonitorProfileAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected when a Task.Delay is cancelled. + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + finally + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Interval Profiling End"); + } + } + } + + /// + /// Executes the Interval monitor profile once. + /// + /// A representing the result of the asynchronous operation. + private Task ExecuteIntervalMonitorProfileAsync(CancellationToken cancellationToken) + { + this.session = null; + return Task.Run(async () => + { + try + { + // creating a unique session name to run simultaneously. + using (this.session = new TraceEventSession($"{this.Provider.Replace(" ", string.Empty)}_{Guid.NewGuid().ToString()}", null)) + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Started Interval Trace Event Session : {this.session.SessionName}"); + + // Enable providers + await this.StartETWTrace(this.ProviderType, this.Provider, this.ProviderKeywords, this.ProviderEvents, this.Filters).ConfigureAwait(false); + + // This is a blocking call. It will end when session is disposed. + Task recordEventsTask = Task.Run(() => this.session.Source.Process()); + + DateTime nextProfilingTime = DateTime.UtcNow; + while (!cancellationToken.IsCancellationRequested) + { + try + { + if (DateTime.UtcNow >= nextProfilingTime) + { + DateTime profilingTime = DateTime.UtcNow.Add(this.ProfilingPeriod); + + while (!cancellationToken.IsCancellationRequested && DateTime.UtcNow < profilingTime) + { + IDictionary data = null; + try + { + if (this.queue.TryTake(out data, 0, cancellationToken)) + { + var eventInfo = new Dictionary(); + foreach (DictionaryEntry entry in data) + { + eventInfo[entry.Key.ToString()] = entry.Value; + } + + string eventId = null; + if (data.Contains("EventID")) + { + eventId = data["EventID"].ToString(); + } + + string eventDescription = null; + if (data.Contains("EventName")) + { + eventDescription = data["EventName"].ToString(); + } + + this.Logger.LogSystemEvent( + "ETW", + "Windows ETW", + eventId, + LogLevel.Information, + EventContext.Persisted(), + null, + eventDescription, + eventInfo); + } + } + catch (InvalidOperationException) + { + // An InvalidOperationException means that Take() was called on a completed collection + } + catch (OperationCanceledException) + { + this.queue.CompleteAdding(); + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + + await Task.Delay(500, cancellationToken).ConfigureAwait(false); + } + + nextProfilingTime = DateTime.UtcNow.Add(this.ProfilingInterval); + } + } + catch (OperationCanceledException) + { + // Expected when a Task.Delay is cancelled. + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, EventContext.Persisted()); + } + finally + { + this.Logger.LogTraceMessage($"[{nameof(WindowsETWMonitor)}] Completed Interval Trace Event Session {this.session.SessionName}"); + // Stop the session after completing + this.session?.Stop(); + + // Close the session and clean up any resources associated with the session. + // It is ok to call this more than once. + if (this.session != null) + { + this.session.Dispose(); + } + } + }, cancellationToken); + } + + /// + /// Start the tracing by enabling the provider + /// + /// provider type Kernel/Nonkernel + /// provider name + /// keywords defined for kernel. This is specific to kernel provider only. + /// events specific to kernel keywords/ kernel provider. + /// filters to applied on the result + /// + private async Task StartETWTrace(string providerType, string provider, List providerKeywords, List providerEvents, Dictionary> filters) + { + this.session.StopOnDispose = true; + // Enable providers based on type + if (providerType == "Kernel") + { + this.session.EnableKernelProvider(await this.GetKeywords(providerKeywords)); + await this.EnableKernelEventHandlers(providerEvents, filters); + } + else + { + this.session.EnableProvider(provider, TraceEventLevel.Always); + this.session.Source.Dynamic.AddCallbackForProviderEvents(null, data => + { + this.ProcessEtwTrace(data, filters); + }); + } + } + + /// + /// Get the list of provider keywords for enabling kernel provider + /// + /// kernel provider keywords defined in the profile + /// returns all the keywords defined + private Task GetKeywords(List providerKeywords) + { + return Task.Run(() => + { + List kernelKeywords = providerKeywords; + KernelTraceEventParser.Keywords flags = KernelTraceEventParser.Keywords.None; + foreach (string kernelProviders in kernelKeywords) + { + KernelTraceEventParser.Keywords keywords; + Enum.TryParse(kernelProviders, out keywords); + if (keywords != KernelTraceEventParser.Keywords.None) + { + flags |= keywords; + } + } + + return flags; + }); + } + + /// + /// Add event handlers for specified events (To capture kernel specific events) + /// + /// Specific kernel events defined in the profile + /// + /// + private Task EnableKernelEventHandlers(List providerEvents, Dictionary> filters) + { + return Task.Run(() => + { + if (providerEvents != null) + { + foreach (string events in providerEvents) + { + // Adding dynamic event handlers for the events. + Type kType = this.session.Source.Kernel.GetType(); + EventInfo kEvent = kType.GetEvent(events); + if (kEvent != null) + { + // Setup function handlers + kEvent.AddEventHandler(this.session.Source.Kernel, (dynamic data) => + { + this.ProcessEtwTrace(data, filters); + }); + } + } + } + else + { + this.session.Source.Kernel.AddCallbackForProviderEvents(null, data => + { + this.ProcessEtwTrace(data, filters); + }); + } + }); + } + + /// + /// Call back routine for start process ETW Trace. + /// + /// Event emitted data + /// filters to apply on data + private void ProcessEtwTrace(dynamic data, Dictionary> filters) + { + if (data != null) + { + lock (this.lockProvider) + { + Task.Delay(1000).GetAwaiter().GetResult(); + bool addToQueue = false; + Dictionary processLog = this.TraceLogger(data); + // Applying filters on result data to add filtered data to the queue + addToQueue = (filters != null) ? processLog.Where(p => + filters.ContainsKey(p.Key.ToLower()) && + filters[p.Key.ToLower()].Contains(p.Value.ToString().ToLower())) + .ToList().Count == filters.Count : true; + + if (addToQueue && !this.queue.IsAddingCompleted) + { + this.queue.Add(processLog); + } + } + } + } + + /// + /// Reads the XML from the ETW data structure and initializes + /// the Key value pairs of the dictionary. + /// + /// ETW data structure + /// List of important values from ETW data structure. + private Dictionary TraceLogger(TraceEvent data) + { + this.ETWData = new Dictionary + { + { "ActivityID", data.ActivityID.ToString() }, + { "ProviderGuid", data.ProviderGuid.ToString() }, + { "ProviderName", data.ProviderName.ToString() }, + { "EventID", data.ID.ToString() == "Illegal" ? "0" : data.ID.ToString() }, + { "Version", data.Version.ToString() }, + { "Level", ((int)data.Level).ToString() }, + { "Task", data.Task.ToString() }, + { "TaskName", data.TaskName.ToString() }, + { "OpCode", data.Opcode.ToString() }, + { "OpCodeName", data.OpcodeName.ToString() }, + { "Keywords", data.Keywords.ToString() == "None" ? "0" : data.Keywords.ToString() }, + { "TimeStamp", data.TimeStamp.ToUniversalTime().ToString("o") }, + { "ProcessID", data.ProcessID.ToString() }, + { "ProcessName", data.ProcessName.ToString() }, + { "ProcessorNumber", data.ProcessorNumber.ToString() }, + { "ThreadID", data.ThreadID.ToString() }, + { "EventName", data.EventName?.Replace("/", string.Empty) }, + }; + + if (data.PayloadNames != null) + { + foreach (string payloadName in data.PayloadNames) + { + var propertyInfo = data.GetType().GetProperty(payloadName); + var payloadValue = propertyInfo?.GetValue(data, null); + if (!this.ETWData.ContainsKey(payloadName)) + { + this.ETWData.Add(payloadName, payloadValue?.ToString() ?? string.Empty); + } + } + } + + return this.ETWData; + } + } +} diff --git a/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMetricsParser.cs b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMetricsParser.cs new file mode 100644 index 0000000000..fe5f48b195 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMetricsParser.cs @@ -0,0 +1,453 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Globalization; + using System.Linq; + using System.Text; + using System.Text.RegularExpressions; + using VirtualClient.Common.Extensions; + using VirtualClient.Contracts; + + /// + /// Parser for 'ipmiutil' output to extract SEL metrics. + /// + public static class IpmiUtilSelMetricsParser + { + // e.g. + // RecId | Date/Time | SEV | Src| Evt_Type | Sensor | Evt_detail + internal static readonly Regex SelHeaderPattern = new Regex( + @"RecId|Evt_Type|Sensor|Evt_detail", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // e.g. + // 0002 | 12/18/23 20:16:51 + internal static readonly Regex SelRecordIdPattern = new Regex( + @"\s*[a-z0-9]+\s*\|\s*\d{2}/\d{2}/\d{4}\s*\d{2}:\d{2}:\d{2}", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // e.g. + // 0002 | 12/18/23 20:16:51 | INF | Sms| OS Critical Stop | na | OS Graceful Shutdown + internal static readonly Regex SelRecordPattern = new Regex( + @"\s*(?[0-9a-z]+)\s*\|\s*(?\d{2}/\d{2}/\d{2}\s*\d{2}:\d{2}:\d{2})\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|*\s*(?[^|]+)*", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // e.g. + // 0002 | 12/18/23 20:16:51 | INF | Sms| OS Critical Stop | na | OS Graceful Shutdown + internal static readonly Regex SelRawRecordPattern = new Regex( + @"^([0-9a-f]{2}\s*)+", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // e.g. + // Predictive failure 6f [02 02 18] -> Predictive failure + private static readonly Regex SelRecordEventDetailPattern = new Regex( + @"([a-z0-9]{3,})", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly object LockObject = new object(); + + /// + /// Combines the raw SEL results with the decoded results. + /// + /// The raw SEL results (e.g. 02 00 02 33 19 81 65 41 00 04 20 00 6f 03 ff ff). + /// The decoded SEL results (e.g. 0002 | 12/18/23 20:16:51 | INF | Sms| OS Critical Stop | na | OS Graceful Shutdown). + /// SEL records with raw and decoded results combined. + public static string CombineRecords(string rawResults, string decodedResults) + { + lock (IpmiUtilSelMetricsParser.LockObject) + { + string combinedResults = decodedResults; + if (!string.IsNullOrWhiteSpace(rawResults) && !string.IsNullOrWhiteSpace(decodedResults)) + { + if (IpmiUtilSelMetricsParser.TryGetDecodedFormatRecords(decodedResults, out IList decodedRecords, out IList headers)) + { + List combinedRecords = new List(); + + // 1) Index the decoded results records. + foreach (string record in decodedRecords) + { + // Format: + // Decoded Record = 0002 | 12/18/23 20:16:51 | INF | Sms| OS Critical Stop | na | OS Graceful Shutdown + // Matching Raw Record = 02 00 02 33 19 81 65 41 00 04 20 00 6f 03 ff ff + // + // Record ID 0002 == 02 00 + Match recordMatch = IpmiUtilSelMetricsParser.SelRecordPattern.Match(record); + if (recordMatch.Success) + { + // Convert the record ID in the decoded format (e.g. 0002) to the format that it + // will be in the raw results (e.g. 02 00). + string recordId = recordMatch.Groups["RecordId"].Value; + string rawRecordId = string.Join(string.Empty, Regex.Split(recordId, "([0-9a-f]{2})", RegexOptions.IgnoreCase).Reverse()); + + // e.g. + // ["0200"] = 0002 | 12/18/23 20:16:51 | INF | Sms| OS Critical Stop | na | OS Graceful Shutdown + combinedRecords.Add(new ExtendedRecord(rawRecordId, record)); + } + } + + if (IpmiUtilSelMetricsParser.TryGetRawFormatRecords(rawResults, out IList rawResultsRecords)) + { + // 2) Step through indexed decoded results and match the decoded result + // with the raw result. + foreach (ExtendedRecord record in combinedRecords) + { + for (int i = 0; i < rawResultsRecords.Count; i++) + { + string rawDetails = rawResultsRecords[i]; + if (IpmiUtilSelMetricsParser.SelRawRecordPattern.IsMatch(rawDetails)) + { + // Format: + // 02 00 02 33 19 81 65 41 00 04 20 00 6f 03 ff ff + // + // The ipmitool represents the record ID in 2 different formats between the + // raw results (e.g. 02 00) and the decoded results (e.g. 0002). The keys in the + // decoded record mapping dictionary have been previously converted to the raw record ID + // format. + if (rawDetails.RemoveWhitespace().StartsWith(record.Id, StringComparison.OrdinalIgnoreCase)) + { + record.CombineWith(rawResultsRecords[i]); + + // Shrinking the set for increasing efficiency (less records) to consider + // each time we match a record. + rawResultsRecords.RemoveAt(i); + break; + } + } + } + } + } + + if (combinedRecords?.Any() == true) + { + StringBuilder finalResults = new StringBuilder(); + foreach (string header in headers) + { + if (!string.IsNullOrWhiteSpace(header)) + { + finalResults.AppendLine(header); + } + + if (IpmiUtilSelMetricsParser.SelHeaderPattern.IsMatch(header)) + { + // Break as soon as we have the column headers row so that the + // SEL records immediately follow. + break; + } + } + + foreach (ExtendedRecord record in combinedRecords) + { + finalResults.AppendLine(record.Details); + } + + combinedResults = finalResults.ToString().Trim(); + } + } + } + + return combinedResults; + } + } + + /// + /// Identifies the set of records that are new or unprocessed since the last record + /// processed. + /// + /// Text containing the ipmiutil SEL record results. + /// The last record processed. This should be include the record ID, date and timestamp from the record (e.g. 1a | 08/07/24 11:55:50). + /// New records to process. + /// True if new records are found. + public static bool TryGetNewRecords(string results, string lastRecordProcessed, out IList newRecords) + { + lock (IpmiUtilSelMetricsParser.LockObject) + { + newRecords = null; + + if (!string.IsNullOrWhiteSpace(results)) + { + if (IpmiUtilSelMetricsParser.TryGetDecodedFormatRecords(results, out IList records, out IList headers)) + { + string recordsToProcess = results; + + if (!string.IsNullOrWhiteSpace(lastRecordProcessed)) + { + // e.g. + // last record process = 000f | 12/18/23 20:16:51 + string[] parts = lastRecordProcessed.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + if (parts?.Count() >= 2) + { + string expression = $@"\s*{parts[0]}\s*\|\s*{parts[1]}[\x20-\x7E]+"; + string[] sections = Regex.Split(results, expression, RegexOptions.IgnoreCase); + + if (sections?.Length == 1) + { + // If the set of results DOES NOT contain a matching record ID, then the full set of records + // will be returned. This happens for example when the SEL log is cleared at some point. + recordsToProcess = sections[0]?.Trim(); + } + else if (sections?.Length == 2) + { + // If the results DOES contain a matching record ID, then the section after that record ID + // represents the new records. + recordsToProcess = sections[1]?.Trim(); + } + } + } + + if (!string.IsNullOrWhiteSpace(recordsToProcess)) + { + IpmiUtilSelMetricsParser.TryGetDecodedFormatRecords(recordsToProcess, out newRecords, out headers); + } + } + } + + return newRecords?.Any() == true; + } + } + + /// + /// Parses metrics from the ipmiutil SEL results/records. + /// + /// The results of the 'ipmiutil sel' command execution. + /// The last record processed in previous 'ipmiutil sel' command results. + /// A filter to apply that determines which events will be processed for metrics. + public static ParsingResult Parse(string results, string lastRecordProcessed = null, Regex eventsFilter = null) + { + try + { + lock (IpmiUtilSelMetricsParser.LockObject) + { + string currentRecordProcessed = null; + IList metrics = new List(); + + if (!string.IsNullOrWhiteSpace(results)) + { + if (IpmiUtilSelMetricsParser.TryGetNewRecords(results, lastRecordProcessed, out IList newRecords)) + { + foreach (string record in newRecords) + { + Match recordMatch = IpmiUtilSelMetricsParser.SelRecordPattern.Match(record); + if (recordMatch.Success && recordMatch.Groups.TryGetValue("RecordId", out Group recordId)) + { + currentRecordProcessed = recordId.Value; + if (eventsFilter != null && !eventsFilter.IsMatch(record)) + { + continue; + } + + if (recordMatch.Groups.TryGetValue("EventDetail", out Group eventDetail) + && recordMatch.Groups.TryGetValue("EventType", out Group eventType) + && recordMatch.Groups.TryGetValue("Source", out Group eventSource) + && recordMatch.Groups.TryGetValue("Severity", out Group severity)) + { + recordMatch.Groups.TryGetValue("Sensor", out Group sensor); + recordMatch.Groups.TryGetValue("EventRawDetail", out Group eventRawDetail); + recordMatch.Groups.TryGetValue("Timestamp", out Group timestamp); + DateTime.TryParseExact(timestamp.Value?.Trim(), "MM/dd/yy HH:mm:ss", null, DateTimeStyles.None, out DateTime eventTimestamp); + + string metricName = IpmiUtilSelMetricsParser.GetMetricName( + eventType.Value?.Trim(), + eventSource.Value?.Trim(), + eventDetail.Value?.Trim(), + sensor?.Value?.Trim(), + eventRawDetail?.Value?.Trim()); + + if (!string.IsNullOrWhiteSpace(metricName)) + { + IDictionary metadata = new Dictionary + { + ["record"] = record?.Trim(), + ["recordId"] = recordId?.Value?.Trim(), + ["severity"] = severity?.Value?.Trim(), + ["sensor"] = sensor?.Value?.Trim(), + ["eventTimestamp"] = eventTimestamp, + ["eventType"] = eventType?.Value?.Trim(), + ["eventSource"] = eventSource?.Value?.Trim(), + ["eventDetail"] = eventDetail?.Value?.Trim(), + ["eventRawDetail"] = eventRawDetail?.Value?.Trim() + }; + + metrics.Add(new Metric( + metricName, + 1, + MetricUnit.Count, + MetricRelativity.Undefined, + tags: new List + { + "IPMI", + eventSource?.Value.Trim(), + severity?.Value?.Trim() + }, + description: $"{severity?.Value} level IPMI SEL event ({record?.Trim()}).", + metadata: metadata)); + } + } + } + } + } + } + + return new ParsingResult(metrics, currentRecordProcessed); + } + } + catch (Exception exc) + { + throw new WorkloadResultsException("Failed to parse SEL metrics from ipmiutil results.", exc, ErrorReason.InvalidResults); + } + } + + private static string GetMetricName(string eventType, string source, string eventDetail = null, string sensor = null, string eventRawDetail = null) + { + // e.g. + // 00ad | 01/19/24 08:48:37 | CRT | EFI | Power Supply | #b3 | Predictive failure 6f [02 02 18] | 02 00 02 33 19 81 65 41 00 04 20 00 6f 03 ff ff + // EFI_Power_Supply_#b3_Predictive_failure_6f_[02_02_18] 02_00_02_33_19_81_65_41_00_04_20_00_6f_03_ff_ff + // + // Format: ___ + // Rules: + // - Only replace whitespace with underscores (keep /, -, [, ], etc.) + // - Include sensor if not "na" or empty + // - Space separator between text and hex + // - Hex bytes joined with underscores + + List nameParts = new List(); + + // Add source and eventType + if (!string.IsNullOrWhiteSpace(source)) + { + nameParts.Add(source); + } + + if (!string.IsNullOrWhiteSpace(eventType)) + { + nameParts.Add(eventType); + } + + // Add sensor if it's not "na" or empty + if (!string.IsNullOrWhiteSpace(sensor) && !sensor.Equals("na", StringComparison.OrdinalIgnoreCase)) + { + nameParts.Add(sensor); + } + + // Add event detail if available + if (!string.IsNullOrWhiteSpace(eventDetail)) + { + nameParts.Add(eventDetail); + } + + // Join all parts and replace only whitespace with underscores + string textPart = string.Join("_", nameParts); + textPart = Regex.Replace(textPart, @"\s+", "_"); + + // Add hex part if available + if (!string.IsNullOrWhiteSpace(eventRawDetail)) + { + // Replace spaces in hex with underscores + string hexPart = eventRawDetail.Replace(" ", "_"); + return $"{textPart} {hexPart}"; + } + + return textPart; + } + + private static bool TryGetDecodedFormatRecords(string results, out IList records, out IList headers) + { + records = null; + headers = null; + + string[] lines = Regex.Split(results, @"\r?\n"); + if (lines?.Any() == true) + { + records = new List(); + headers = new List(); + + foreach (string line in lines) + { + if (IpmiUtilSelMetricsParser.SelRecordPattern.IsMatch(line)) + { + records.Add(line); + } + else if (!string.IsNullOrWhiteSpace(line)) + { + headers.Add(line); + } + } + } + + return records?.Any() == true; + } + + private static bool TryGetRawFormatRecords(string results, out IList records) + { + records = null; + + string[] lines = Regex.Split(results, @"\r?\n"); + if (lines?.Any() == true) + { + records = new List(); + + foreach (string line in lines) + { + if (IpmiUtilSelMetricsParser.SelRawRecordPattern.IsMatch(line)) + { + records.Add(line); + } + } + } + + return records?.Any() == true; + } + + /// + /// Represents the result of the metrics parsing. + /// + public class ParsingResult + { + /// + /// Initializes a new instance of the class. + /// + public ParsingResult(IEnumerable metrics, string lastRecordProcessed = null) + { + this.LastRecordProcessed = lastRecordProcessed; + + if (metrics?.Any() == true) + { + this.Metrics = new List(metrics); + } + } + + /// + /// The last SEL record processed (e.g. 00e8 | 02/19/24 18:42:00). + /// + public string LastRecordProcessed { get; } + + /// + /// Metrics parsed from the SEL results. + /// + public IList Metrics { get; } + } + + [DebuggerDisplay("{Id} = {Details}")] + private class ExtendedRecord + { + public ExtendedRecord(string id, string details) + { + this.Id = id; + this.Details = details; + } + + public string Id { get; } + + public string Details { get; set; } + + public void CombineWith(string rawData) + { + this.Details = $"{this.Details} | {rawData}"; + } + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMonitor.cs b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMonitor.cs new file mode 100644 index 0000000000..b53febd805 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSelMonitor.cs @@ -0,0 +1,1463 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// Executes the ipmiutil toolset to monitor SEL logs on the system. + /// Enhanced to detect suspected hardware errors. + /// + /// + /// https://ipmiutil.sourceforge.net/ + /// https://ipmiutil.sourceforge.net/docs/UserGuide + /// + [SupportedPlatforms("linux-x64,linux-arm64,win-x64,win-arm64")] + public class IpmiUtilSelMonitor : VirtualClientIntervalBasedMonitor + { + internal static readonly IDictionary ToolsetAlternateLocations = new Dictionary + { + // The set of possible locations of the ipmiutil toolset for the case that + // the host monitoring package is not installed. + { "C:\\BladeFX_latest\\BladeFX\\Tools\\IpmiUtil\\ipmiutil.exe", PlatformID.Win32NT } + }; + + // Specific hardware error patterns + internal static readonly string[] HardwareErrorPatterns = new[] + { + "failure detected", "fault", "error", "throttled", "power supply", + "temperature", "voltage", "fan", "memory", "processor" + }; + + private const string CommandToGetDecodedSel = "sel -uc"; + private const string CommandToGetRawHexCodes = "sel -r"; + private const string CommandToClearSelLog = "sel -d"; + + private IFileSystem fileSystem; + private ProcessManager processManager; + private ISystemManagement systemManager; + private string lastRecordProcessed; + + /// + /// Initializes a new instance of the class. + /// + public IpmiUtilSelMonitor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.fileSystem = dependencies.GetService(); + this.processManager = dependencies.GetService(); + this.systemManager = dependencies.GetService(); + } + + /// + /// Option to clear the SEL data. + /// + public bool ClearSel + { + get + { + return this.Parameters.GetValue(nameof(this.ClearSel), false); + } + + set + { + this.Parameters[nameof(this.ClearSel)] = value; + } + } + + /// + /// Option to enable suspected event detection and create separate files for suspected hardware errors. + /// When enabled, creates additional output files containing only suspected events. + /// + public bool CaptureSuspectedEvents + { + get + { + return this.Parameters.GetValue(nameof(this.CaptureSuspectedEvents), false); + } + + set + { + this.Parameters[nameof(this.CaptureSuspectedEvents)] = value; + } + } + + /// + /// Option to include warning level events in suspected event detection. + /// When disabled, only hardware errors and critical events are considered suspected. + /// + public bool IncludeWarningsInSuspectedEvents + { + get + { + return this.Parameters.GetValue(nameof(this.IncludeWarningsInSuspectedEvents), true); + } + + set + { + this.Parameters[nameof(this.IncludeWarningsInSuspectedEvents)] = value; + } + } + + /// + /// Option to enable SEL decoding. When enabled, the raw SEL hex data + /// is passed through the configured decoder tool for enhanced, human-readable decoding. + /// + public bool EnableSelDecoding + { + get + { + return this.Parameters.GetValue(nameof(this.EnableSelDecoding), false); + } + + set + { + this.Parameters[nameof(this.EnableSelDecoding)] = value; + } + } + + /// + /// Selects the decoder engine to use. Supported values: "Sherlock", "Generic". + /// Required when is true. + /// + public string DecoderType + { + get + { + return this.Parameters.GetValue(nameof(this.DecoderType)); + } + + set + { + this.Parameters[nameof(this.DecoderType)] = value; + } + } + + /// + /// Additional command-line arguments to pass to the SEL decoder. + /// For Sherlock, these are appended before the input file (-f) argument. + /// For Generic, these are appended after the input file argument. + /// e.g. "-nc -t Pacific -v --sel_parser" or "--sku c41a8". + /// + public string DecoderArguments + { + get + { + this.Parameters.TryGetValue(nameof(this.DecoderArguments), out IConvertible decoderArguments); + return decoderArguments?.ToString(); + } + + set + { + this.Parameters[nameof(this.DecoderArguments)] = value; + } + } + + /// + /// The name of the package that contains the SEL decoder toolset. + /// + public string DecoderPackageName + { + get + { + this.Parameters.TryGetValue(nameof(this.DecoderPackageName), out IConvertible packageName); + return packageName?.ToString(); + } + + set + { + this.Parameters[nameof(this.DecoderPackageName)] = value; + } + } + + /// + /// The executable name of the SEL decoder tool inside the decoder package. + /// Optional. If not provided, defaults to "crcsdkseldecoder.exe" for Generic + /// and "sel_decode.exe" for Sherlock. Profiles may override this when the + /// package ships a different executable name. + /// + public string DecoderExecutableName + { + get + { + this.Parameters.TryGetValue(nameof(this.DecoderExecutableName), out IConvertible executableName); + return executableName?.ToString(); + } + + set + { + this.Parameters[nameof(this.DecoderExecutableName)] = value; + } + } + + /// + /// A regular expression that will be used to match records on each line of the + /// output. Default is to match any events with severity level major/MAJ or critical/CRT. + /// + public string EventsFilter + { + get + { + return this.Parameters.GetValue(nameof(this.EventsFilter), "MAJ|CRT"); + } + + set + { + this.Parameters[nameof(this.EventsFilter)] = value; + } + } + + /// + /// The count of Decoded SEL to be emitted as a metric. This is useful to track how many SEL entries were decoded for each monitor iteration. + /// + public int DecodedSelCount { get; set; } + + /// + /// The path to the impiutil + /// + protected string IpmiUtilExePath { get; set; } + + /// + /// The path to the SEL decoder executable (Sherlock or Generic, based on ). + /// + protected string DecoderExePath { get; set; } + + /// + /// Initializes the monitor. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + string executableName = this.Platform == PlatformID.Unix ? "ipmiutil" : "ipmiutil.exe"; + string helpSwitch = this.Platform == PlatformID.Unix ? "--help" : "/?"; + + DependencyPath monitorsPackage = null; + + if (this.Parameters.TryGetValue(nameof(this.PackageName), out IConvertible packageName) + && !string.IsNullOrWhiteSpace(packageName?.ToString())) + { + try + { + DependencyPath toolsetPackage = await this.GetPlatformSpecificPackageAsync(packageName?.ToString(), cancellationToken, throwIfNotfound: false); + + // Account for shortcomings in the GetPlatformSpecificPackageAsync method + // implementation when throwIfNotFound = false until fixed in open source. + if (toolsetPackage != null && this.fileSystem.File.Exists(this.Combine(toolsetPackage.Path, executableName))) + { + monitorsPackage = toolsetPackage; + } + } + catch + { + // Account for bug in GetPlatformSpecificPackageAsync method when throwIfNotFound = false + // until the bug is addressed. + } + } + + if (monitorsPackage != null) + { + // e.g. + // C:\Users\AnyUser\VirtualClient\packages\host.monitors.1.0.0\win-x64\ipmiutil.exe + // /home/anyuser/VirtualClient/packages/host.monitors.1.0.0/linux-x64/ipmiutil + this.IpmiUtilExePath = this.Combine(monitorsPackage.Path, executableName); + + if (this.Platform == PlatformID.Unix) + { + await this.systemManager.MakeFileExecutableAsync(this.IpmiUtilExePath, this.Platform, cancellationToken); + } + } + else + { + // If we cannot find the ipmiutil toolset in the alternate locations, it might be available + // in a location defined in the PATH environment variable. + // + // e.g. + // ipmiutil --help + // ipmiutil /? + using (IProcessProxy process = this.processManager.CreateProcess(executableName, helpSwitch, null)) + { + try + { + await process.StartAndWaitAsync(cancellationToken); + + // ipmiutil returns various exit codes on success when using the help + // switch. We check for the presence of standard output to confirm the toolset exists. + if (!cancellationToken.IsCancellationRequested && !string.IsNullOrWhiteSpace(process.StandardOutput?.ToString())) + { + this.IpmiUtilExePath = executableName; + } + } + catch + { + // Win32Exception will be thrown if the executable does not exist on the + // system at all. + } + } + + // If we cannot find the ipmiutil toolset in any other location, fallback to any defined + // alternate locations. + if (string.IsNullOrWhiteSpace(this.IpmiUtilExePath)) + { + foreach (var alternateLocation in IpmiUtilSelMonitor.ToolsetAlternateLocations.Where(entry => entry.Value == this.Platform)) + { + string executablePath = alternateLocation.Key; + if (this.fileSystem.File.Exists(executablePath)) + { + this.IpmiUtilExePath = executablePath; + break; + } + } + } + } + + if (this.EnableSelDecoding) + { + await this.InitializeDecoderAsync(telemetryContext, cancellationToken); + } + } + catch (Exception exc) + { + // Do not let the monitor crash the application. + this.Logger.LogMessage( + $"{this.TypeName}.InitializationError", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + } + + /// + /// Monitors Ipmiutil on the target machine + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return Task.Run(async () => + { + try + { + if (string.IsNullOrWhiteSpace(this.IpmiUtilExePath)) + { + this.Logger.LogMessage( + $"The 'ipmiutil' toolset was not found on the system. This monitor will exit.", + LogLevel.Warning, + telemetryContext); + } + else + { + // All background monitor ExecuteAsync methods should be either 'async' or should use a Task.Run() if running a 'while' loop or the + // logic will block without returning. Monitors are typically expected to be fire-and-forget. + var commandArguments = string.Join(",", + IpmiUtilSelMonitor.CommandToGetDecodedSel, + IpmiUtilSelMonitor.CommandToGetRawHexCodes, + this.ClearSel ? IpmiUtilSelMonitor.CommandToClearSelLog : string.Empty).TrimEnd(','); + + long currentIteration = 1; + DateTime nextIteration = DateTime.UtcNow.Add(this.MonitorWarmupPeriod); + while (!cancellationToken.IsCancellationRequested && !this.IsIterationComplete(currentIteration)) + { + try + { + string combinedSelRecords = null; + string rawSelRecords = null; + await this.WaitAsync(nextIteration, cancellationToken); + nextIteration = DateTime.UtcNow.Add(this.MonitorFrequency); + + if (cancellationToken.IsCancellationRequested) + { + break; + } + + // 1) Decoded (human-readable) SEL data + // + // e.g. + // 07cc | 01/24/25 21:35:33 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C + string decodedSelRecords = null; + DateTime startTime = DateTime.UtcNow; + + using (IProcessProxy process1 = await this.ExecuteCommandAsync(this.IpmiUtilExePath, "sel -uc", null, telemetryContext, CancellationToken.None, runElevated: true)) + { + if (process1.ExitCode != 0) + { + await this.LogProcessDetailsAsync(process1, telemetryContext, "ipmiutil_sel_error"); + + this.Logger.LogMessage( + $"{this.TypeName}.ReadDecodedSelError", + LogLevel.Warning, + telemetryContext.Clone().AddProcessDetails(process1.ToProcessDetails("ipmiutil"))); + + continue; + } + + decodedSelRecords = process1.StandardOutput?.ToString(); + if (!string.IsNullOrWhiteSpace(decodedSelRecords)) + { + await this.WaitAsync(TimeSpan.FromSeconds(2), CancellationToken.None); + + // 2) Raw ASCII SEL data + // + // e.g. + // cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b + // + // Note that the decoded record and ascii record will be paired together when matched + // by record ID (e.g. record ID = 07cc -> cc 07). The hash at the beginning is used to identify + // unique records so that the logic is prepared to handle cases where SEL logs are or are not + // cleared over periods of time. + // + // RecId | Date/Time | SEV | Src | Evt_Type | Sensor | Evt_detail | Supplementary/Added -> Evt_detail_raw + // 07cc | 01/24/25 21:35:33 | MAJ | BMC | Temperature | GPU0_DRAM0_TMP0 | Hi Crit thresh actual=127.00 C, threshold=107.00 C | cc 07 02 a5 07 94 67 20 00 04 01 5a 01 59 7f 6b + + using (IProcessProxy process2 = await this.ExecuteCommandAsync(this.IpmiUtilExePath, "sel -r", null, telemetryContext, CancellationToken.None, runElevated: true)) + { + if (process2.ExitCode != 0) + { + await this.LogProcessDetailsAsync(process2, telemetryContext, "ipmiutil_sel_error"); + + this.Logger.LogMessage( + $"{this.TypeName}.ReadRawSelError", + LogLevel.Warning, + telemetryContext.Clone().AddProcessDetails(process2.ToProcessDetails("ipmiutil"))); + } + else + { + rawSelRecords = process2.StandardOutput?.ToString(); + combinedSelRecords = IpmiUtilSelMetricsParser.CombineRecords(rawSelRecords, decodedSelRecords); + } + } + } + + DateTime exitTime = DateTime.UtcNow; + await this.LogProcessDetailsAsync( + new ProcessDetails + { + Id = process1.Id, + ExitCode = process1.ExitCode, + CommandLine = process1.FullCommand(), + StandardOutput = combinedSelRecords, + StandardError = process1.StandardError?.ToString(), + ToolName = "ipmiutil_sel", + WorkingDirectory = Environment.CurrentDirectory, + StartTime = startTime, + ExitTime = exitTime + }, + telemetryContext, + logFileName: "ipmiutil_sel"); + } + + this.CaptureMetrics(combinedSelRecords ?? decodedSelRecords, commandArguments, startTime, DateTime.UtcNow, telemetryContext); + + // Capture suspected events if enabled + if (this.CaptureSuspectedEvents) + { + await this.CaptureSuspectedEventsAsync(combinedSelRecords ?? decodedSelRecords, telemetryContext, cancellationToken); + } + + // Run SEL decoding if enabled. The decoder engine (Sherlock or Generic) is selected via DecoderType. + if (this.EnableSelDecoding && !string.IsNullOrWhiteSpace(this.DecoderExePath)) + { + if (string.Equals(this.DecoderType, "Generic", StringComparison.OrdinalIgnoreCase)) + { + // The generic decoder natively handles pure hex lines (the format produced by `sel -r`), + // so the raw hex records are passed directly without the combined/decoded format. + await this.ExecuteGenericDecodingAsync(rawSelRecords, telemetryContext, cancellationToken); + } + else if (string.Equals(this.DecoderType, "Sherlock", StringComparison.OrdinalIgnoreCase)) + { + await this.ExecuteSherlockDecodingAsync(combinedSelRecords ?? decodedSelRecords, telemetryContext, cancellationToken); + } + else + { + throw new ArgumentException($"Unsupported DecoderType: '{this.DecoderType}'. Supported values are 'Sherlock' and 'Generic'."); + } + } + + if (this.ClearSel) + { + using (IProcessProxy process3 = await this.ExecuteCommandAsync(this.IpmiUtilExePath, "sel -d", null, telemetryContext, CancellationToken.None, runElevated: true)) + { + await this.LogProcessDetailsAsync(process3, telemetryContext, "ipmiutil_sel_clear"); + + if (process3.ExitCode != 0) + { + this.Logger.LogMessage( + $"{this.TypeName}.ClearSelError", + LogLevel.Warning, + telemetryContext.Clone().AddProcessDetails(process3.ToProcessDetails("ipmiutil"))); + } + } + } + } + catch (Exception exc) + { + // Do not let the monitor crash the application. Continue to attempt to run the + // ipmiutil monitoring commands. + this.Logger.LogMessage( + $"{this.TypeName}.{this.Scenario}Warning", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + finally + { + currentIteration++; + } + } + } + } + catch (Exception exc) + { + // Do not let the monitor crash the application. + this.Logger.LogMessage( + $"{this.TypeName}.ExecuteError", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + }); + } + + /// + /// Capture metrics from the results of the 'ipmiutil sel' command. + /// + protected void CaptureMetrics(string results, string commandArguments, DateTime startTime, DateTime endTime, EventContext telemetryContext) + { + if (!string.IsNullOrWhiteSpace(results)) + { + Regex eventsFilter = null; + if (!string.IsNullOrWhiteSpace(this.EventsFilter)) + { + eventsFilter = new Regex(this.EventsFilter, RegexOptions.IgnoreCase); + } + + IpmiUtilSelMetricsParser.ParsingResult parsingResult = IpmiUtilSelMetricsParser.Parse( + results, + this.lastRecordProcessed, + eventsFilter); + + if (parsingResult?.Metrics?.Any() == true) + { + this.Logger.LogMetrics( + toolName: "ipmiutil", + scenarioName: "SEL", + startTime, + endTime, + parsingResult.Metrics, + metricCategorization: "IPMI", + scenarioArguments: commandArguments, + this.Tags, + telemetryContext); + } + + this.lastRecordProcessed = parsingResult.LastRecordProcessed; + } + } + + /// + /// Initializes the SEL decoder toolset based on the configured . + /// + protected async Task InitializeDecoderAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + string executableName; + if (!string.IsNullOrWhiteSpace(this.DecoderExecutableName)) + { + executableName = this.DecoderExecutableName; + } + else if (string.Equals(this.DecoderType, "Generic", StringComparison.OrdinalIgnoreCase)) + { + executableName = this.Platform == PlatformID.Unix ? "crcsdkseldecoder" : "crcsdkseldecoder.exe"; + } + else if (string.Equals(this.DecoderType, "Sherlock", StringComparison.OrdinalIgnoreCase)) + { + executableName = "sel_decode.exe"; + } + else + { + throw new ArgumentException($"Unsupported DecoderType: '{this.DecoderType}'. Supported values are 'Sherlock' and 'Generic'."); + } + + if (!string.IsNullOrWhiteSpace(this.DecoderPackageName)) + { + try + { + DependencyPath decoderPackage = await this.GetPlatformSpecificPackageAsync(this.DecoderPackageName, cancellationToken, throwIfNotfound: false); + + if (decoderPackage != null && this.fileSystem.File.Exists(this.Combine(decoderPackage.Path, executableName))) + { + this.DecoderExePath = this.Combine(decoderPackage.Path, executableName); + + if (this.Platform == PlatformID.Unix) + { + await this.systemManager.MakeFileExecutableAsync(this.DecoderExePath, this.Platform, cancellationToken); + } + } + } + catch + { + // Account for bug in GetPlatformSpecificPackageAsync method when throwIfNotFound = false + // until the bug is addressed. + } + } + + if (string.IsNullOrWhiteSpace(this.DecoderExePath)) + { + this.Logger.LogMessage( + $"{this.TypeName}.DecoderToolsetNotFound", + LogLevel.Warning, + telemetryContext.Clone().AddContext("message", $"The {this.DecoderType} SEL decoder toolset was not found. Decoding will be skipped.") + .AddContext("decoderType", this.DecoderType) + .AddContext("decoderPackageName", this.DecoderPackageName)); + } + } + catch (ArgumentException) + { + throw; + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.DecoderInitializationError", + LogLevel.Warning, + telemetryContext.Clone().AddContext("decoderType", this.DecoderType).AddError(exc)); + } + } + + /// + /// Executes the Sherlock SEL decoder and uploads the decoded output. + /// Create a file even if no data found and a comment in the file stating no sel records available for decoding. + /// + protected async Task ExecuteSherlockDecodingAsync(string selRecords, EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + if (string.IsNullOrWhiteSpace(selRecords)) + { + this.Logger.LogMessage( + $"{this.TypeName}.SherlockDecodingSkipped", + LogLevel.Information, + telemetryContext.Clone().AddContext("message", "No sel data available for Sherlock decoding.")); + return; + } + + this.DecodedSelCount = 0; + string logsDirectory = this.PlatformSpecifics.GetLogsPath(); + string scenarioDirectory = this.PlatformSpecifics.Combine(logsDirectory, this.Scenario ?? "IpmiUtilSel"); + this.fileSystem.Directory.CreateDirectory(scenarioDirectory); + + string timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd-HH-mm-ss"); + string inputFilePath = this.PlatformSpecifics.Combine(scenarioDirectory, $"{timestamp}_selRecords.txt"); + await this.fileSystem.File.WriteAllTextAsync(inputFilePath, selRecords, cancellationToken); + + // Execute the Sherlock decoder. + string extraArgs = !string.IsNullOrWhiteSpace(this.DecoderArguments) ? $"{this.DecoderArguments.Trim()} " : string.Empty; + string decoderArguments = $"{extraArgs} -f \"{inputFilePath}\""; + + using (IProcessProxy sherlockProcess = await this.ExecuteCommandAsync( + this.DecoderExePath, decoderArguments, null, telemetryContext, CancellationToken.None, runElevated: false)) + { + await this.LogProcessDetailsAsync( + sherlockProcess, + telemetryContext, + toolName: "ipmiutil_sel", + logFileName: "decodedseloutput"); + + if (sherlockProcess.ExitCode == 0 && !string.IsNullOrWhiteSpace(sherlockProcess.StandardOutput.ToString())) + { + this.DecodedSelCount = this.ParseAndLogDecodedSelEntries(sherlockProcess.StandardOutput.ToString(), telemetryContext); + } + else + { + this.Logger.LogMessage( + $"{this.TypeName}.SherlockDecodingError", + LogLevel.Warning, + telemetryContext.Clone().AddProcessDetails(sherlockProcess.ToProcessDetails("sel_decode"))); + } + + this.Logger.LogMetric( + new Metric(nameof(this.DecodedSelCount), this.DecodedSelCount, "count"), + "SherlockIpmiUtilSelDecoder", + this.Scenario, + sherlockProcess.StartTime, + sherlockProcess.ExitTime, + decoderArguments, + telemetryContext); + } + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.SherlockDecodingError", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + } + + /// + /// Saves the raw SEL hex records to a file and executes the generic SEL decoder against it. + /// The generic decoder writes its output to files in a timestamped output directory; the + /// "All-BMC-SEL-Decoded.log" file is read back, parsed, and uploaded to the content store. + /// + protected async Task ExecuteGenericDecodingAsync(string selRecords, EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + if (string.IsNullOrWhiteSpace(selRecords)) + { + this.Logger.LogMessage( + $"{this.TypeName}.GenericDecodingSkipped", + LogLevel.Information, + telemetryContext.Clone().AddContext("message", "No SEL data available for generic decoding.")); + return; + } + + this.DecodedSelCount = 0; + string logsDirectory = this.PlatformSpecifics.GetLogsPath(); + string scenarioDirectory = this.PlatformSpecifics.Combine(logsDirectory, this.Scenario ?? "IpmiUtilSel"); + this.fileSystem.Directory.CreateDirectory(scenarioDirectory); + + string timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd-HH-mm-ss-fff"); + string outputDirectory = this.PlatformSpecifics.Combine(scenarioDirectory, $"{timestamp}_output"); + this.fileSystem.Directory.CreateDirectory(outputDirectory); + + string inputFilePath = this.PlatformSpecifics.Combine(outputDirectory, $"{timestamp}_input_selRecords.txt"); + await this.fileSystem.File.WriteAllTextAsync(inputFilePath, selRecords, cancellationToken); + + string extraArgs = !string.IsNullOrWhiteSpace(this.DecoderArguments) ? $" {this.DecoderArguments.Trim()}" : string.Empty; + string genericArguments = $"\"{inputFilePath}\" -o \"{outputDirectory}\"{extraArgs}"; + + using (IProcessProxy genericProcess = await this.ExecuteCommandAsync( + this.DecoderExePath, genericArguments, null, telemetryContext, CancellationToken.None, runElevated: false)) + { + await this.LogProcessDetailsAsync( + genericProcess, + telemetryContext, + toolName: "ipmiutil_sel", + logFileName: "decodedseloutput"); + + string decodedOutputFile = this.PlatformSpecifics.Combine(outputDirectory, "All-BMC-SEL-Decoded.log"); + + if (genericProcess.ExitCode == 0 && this.fileSystem.File.Exists(decodedOutputFile)) + { + string decodedOutput = await this.fileSystem.File.ReadAllTextAsync(decodedOutputFile, cancellationToken); + + if (!string.IsNullOrWhiteSpace(decodedOutput)) + { + this.DecodedSelCount = this.ParseAndLogGenericDecodedSelEntries(decodedOutput, telemetryContext); + + // Copy the decoded file to a timestamped name and request upload to the content store. + string decodedTimestampedName = $"{timestamp}_decoded_selRecords.log"; + string decodedTimestampedPath = this.PlatformSpecifics.Combine(outputDirectory, decodedTimestampedName); + this.fileSystem.File.Copy(decodedOutputFile, decodedTimestampedPath, overwrite: true); + + if (this.fileSystem.File.Exists(decodedTimestampedPath) && this.TryGetContentStoreManager(out IBlobManager blobManager)) + { + FileUploadDescriptor descriptor = this.CreateFileUploadDescriptor( + new FileContext( + this.fileSystem.FileInfo.New(decodedTimestampedPath), + HttpContentType.PlainText, + Encoding.UTF8.WebName, + this.ExperimentId, + this.AgentId, + "ipmiutil_sel", + this.Scenario, + null, + this.Roles?.FirstOrDefault()), + this.Parameters, + timestamped: false); + + await this.RequestFileUploadAsync(descriptor); + } + } + else + { + this.Logger.LogTraceMessage($"Generic SEL decoder produced an empty output file: {decodedOutputFile}"); + + this.Logger.LogMessage( + $"{this.TypeName}.GenericDecodingError", + LogLevel.Warning, + telemetryContext.Clone() + .AddContext("message", "Decoded output file is empty.") + .AddContext("inputFile", inputFilePath) + .AddContext("outputFile", decodedOutputFile) + .AddProcessDetails(genericProcess.ToProcessDetails("crcsdkseldecoder"))); + } + } + else + { + this.Logger.LogTraceMessage($"Generic SEL decoder failed (exitCode={genericProcess.ExitCode}): {genericProcess.StandardError?.ToString()}"); + + this.Logger.LogMessage( + $"{this.TypeName}.GenericDecodingError", + LogLevel.Warning, + telemetryContext.Clone() + .AddContext("message", genericProcess.StandardError?.ToString()) + .AddContext("exitCode", genericProcess.ExitCode) + .AddContext("inputFile", inputFilePath) + .AddProcessDetails(genericProcess.ToProcessDetails("crcsdkseldecoder"))); + } + + this.Logger.LogMetric( + new Metric(nameof(this.DecodedSelCount), this.DecodedSelCount, "count"), + "GenericIpmiUtilSelDecoder", + this.Scenario, + genericProcess.StartTime, + genericProcess.ExitTime, + genericArguments, + telemetryContext); + } + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.GenericDecodingError", + LogLevel.Warning, + telemetryContext.Clone() + .AddContext("message", exc.Message) + .AddError(exc)); + } + } + + /// + /// Captures suspected events from SEL records. + /// Creates separate files for suspected and warning SEL entries. + /// + protected async Task CaptureSuspectedEventsAsync(string results, EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + if (string.IsNullOrWhiteSpace(results)) + { + // Create empty suspected events file to indicate monitoring ran but found no events + await this.WriteSuspectedEventsFile( + "# No SEL events found during monitoring period\n" + + $"# Monitoring time: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss}\n" + + $"# Suspected event detection: Enabled\n" + + $"# Include warnings: {this.IncludeWarningsInSuspectedEvents}\n", + "SuspectSelEntries", + cancellationToken); + return; + } + + var (suspectedEvents, warningEvents) = this.AnalyzeSuspectedSelEvents(results); + + // Create suspected events file (always create, even if empty) + string suspectedContent = suspectedEvents.Any() + ? string.Join("\n", suspectedEvents) + : "# No suspected SEL events found during monitoring period\n" + + $"# Monitoring time: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss}\n" + + $"# Total SEL events scanned: {this.CountSelRecords(results)}\n" + + $"# Suspected event detection: Enabled\n" + + $"# Include warnings: {this.IncludeWarningsInSuspectedEvents}\n"; + + await this.WriteSuspectedEventsFile(suspectedContent, "SuspectSelEntries", cancellationToken); + + // Create warning events file if there are any warning events + if (warningEvents.Any()) + { + string warningContent = $"# Warning SEL events found during monitoring period\n" + + $"# Monitoring time: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss}\n" + + $"# Warning events count: {warningEvents.Count}\n\n" + + string.Join("\n", warningEvents); + + await this.WriteSuspectedEventsFile(warningContent, "WarningSelEntries", cancellationToken); + } + + // Log telemetry about suspected events + telemetryContext.AddContext("SuspectedSelEventsCount", suspectedEvents.Count); + telemetryContext.AddContext("WarningSelEventsCount", warningEvents.Count); + + if (suspectedEvents.Any()) + { + this.Logger.LogMessage( + $"{this.TypeName}.SuspectedSelEventsFound", + LogLevel.Warning, + telemetryContext.Clone().AddContext("message", $"Found {suspectedEvents.Count} suspected SEL events")); + } + else + { + this.Logger.LogMessage( + $"{this.TypeName}.NoSuspectedSelEventsFound", + LogLevel.Information, + telemetryContext.Clone().AddContext("message", "No suspected SEL events found")); + } + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.SuspectedEventsError", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + } + + /// + /// Analyzes SEL events to identify suspected hardware errors and warnings. + /// + protected (IList suspectedEvents, IList warningEvents) AnalyzeSuspectedSelEvents(string results) + { + var suspectedEvents = new List(); + var warningEvents = new List(); + var assertedEvents = new Dictionary>(); + + if (string.IsNullOrWhiteSpace(results)) + { + return (suspectedEvents, warningEvents); + } + + string[] lines = results.Split('\n', StringSplitOptions.RemoveEmptyEntries); + + foreach (string line in lines) + { + // Skip header lines + if (line.Contains("RecId") && line.Contains("Date/Time")) + { + continue; + } + + var selRecord = this.ParseSelRecord(line); + if (selRecord == null) + { + continue; + } + + bool isSuspected = this.IsSelEventSuspected(selRecord); + bool isWarning = this.IsSelEventWarning(selRecord); + + if (isSuspected) + { + suspectedEvents.Add(line); + } + else if (isWarning) + { + warningEvents.Add(line); + } + } + + return (suspectedEvents, warningEvents); + } + + /// + /// Determines if a SEL event should be considered suspected. + /// + protected bool IsSelEventSuspected(SelRecord selRecord) + { + if (selRecord == null) + { + return false; + } + + // Hardware errors are always suspected + if (selRecord.IsHardwareError) + { + return true; + } + + // Critical and major severity events are always suspected + if (selRecord.Severity?.Equals("CRT", StringComparison.OrdinalIgnoreCase) == true || + selRecord.Severity?.Equals("MAJ", StringComparison.OrdinalIgnoreCase) == true) + { + return true; + } + + // Specific hardware-related events that are always suspected + var suspectedKeywords = new[] + { + "failure detected", "throttled", "power supply ac lost", "fault", "error", + "correctable ecc", "uncorrectable ecc", "ierr", "frb2", "frb3", "predictive failure", + "bus error", "fatal nmi" + }; + + string eventDetail = selRecord.EventDetail?.ToLowerInvariant() ?? string.Empty; + string eventType = selRecord.EventType?.ToLowerInvariant() ?? string.Empty; + + return suspectedKeywords.Any(keyword => + eventDetail.Contains(keyword) || eventType.Contains(keyword)); + } + + /// + /// Determines if a SEL event should be considered a warning that needs pairing analysis. + /// + protected bool IsSelEventWarning(SelRecord selRecord) + { + if (selRecord == null || !this.IncludeWarningsInSuspectedEvents) + { + return false; + } + + // Warning level events that might have asserted/deasserted pairs + string eventDetail = selRecord.EventDetail?.ToLowerInvariant() ?? string.Empty; + + return eventDetail.Contains("asserted") || + eventDetail.Contains("deasserted") || + eventDetail.Contains("exceeded") || + eventDetail.Contains("within") || + eventDetail.Contains("above") || + eventDetail.Contains("below"); + } + + /// + /// Parses a SEL record line into a structured object. + /// + protected SelRecord ParseSelRecord(string line) + { + if (string.IsNullOrWhiteSpace(line)) + { + return null; + } + + var match = IpmiUtilSelMetricsParser.SelRecordPattern.Match(line); + if (!match.Success) + { + return null; + } + + return new SelRecord + { + RecordId = match.Groups["RecordId"]?.Value?.Trim(), + Timestamp = match.Groups["Timestamp"]?.Value?.Trim(), + Severity = match.Groups["Severity"]?.Value?.Trim(), + Source = match.Groups["Source"]?.Value?.Trim(), + EventType = match.Groups["EventType"]?.Value?.Trim(), + Sensor = match.Groups["Sensor"]?.Value?.Trim(), + EventDetail = match.Groups["EventDetail"]?.Value?.Trim(), + IsHardwareError = this.DetermineIfHardwareError(match.Groups["Severity"]?.Value?.Trim(), + match.Groups["EventDetail"]?.Value?.Trim()) + }; + } + + /// + /// Determines if an event represents a hardware error. + /// + protected bool DetermineIfHardwareError(string severity, string eventDetail) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return false; + } + + // Critical and Major events are considered hardware errors + if (severity.Equals("CRT", StringComparison.OrdinalIgnoreCase) || + severity.Equals("MAJ", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + // For other severity levels, check if eventDetail contains hardware error patterns + if (string.IsNullOrWhiteSpace(eventDetail)) + { + return false; + } + + string detail = eventDetail.ToLowerInvariant(); + return HardwareErrorPatterns.Any(pattern => detail.Contains(pattern)); + } + + /// + /// Counts the number of SEL records in the results. + /// + protected int CountSelRecords(string results) + { + if (string.IsNullOrWhiteSpace(results)) + { + return 0; + } + + return results.Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Count(line => IpmiUtilSelMetricsParser.SelRecordPattern.IsMatch(line)); + } + + /// + /// Writes suspected events to a file in the logs directory. + /// + protected async Task WriteSuspectedEventsFile(string content, string fileName, CancellationToken cancellationToken) + { + try + { + string logsDirectory = this.PlatformSpecifics.GetLogsPath(); + string scenarioDirectory = this.PlatformSpecifics.Combine(logsDirectory, this.Scenario ?? "IpmiUtilSel"); + this.fileSystem.Directory.CreateDirectory(scenarioDirectory); + + string filePath = this.PlatformSpecifics.Combine(scenarioDirectory, $"{fileName}-{DateTime.UtcNow:yyyy-MM-dd-HH-mm-ss}.log"); + await this.fileSystem.File.WriteAllTextAsync(filePath, content, cancellationToken); + + await this.UploadEventLogs(filePath); + + this.Logger.LogMessage( + $"{this.TypeName}.SuspectedEventsFileCreated", + LogLevel.Information, + EventContext.Persisted().AddContext("filePath", filePath)); + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.SuspectedEventsFileError", + LogLevel.Warning, + EventContext.Persisted().AddError(exc)); + } + } + + /// + /// Writes the events to a local file and uploads to content store + /// + protected async Task UploadEventLogs(string outputFilePath) + { + // Upload log file to view on result explorer and portal + if (this.TryGetContentStoreManager(out IBlobManager blobManager)) + { + FileUploadDescriptor descriptor = this.CreateFileUploadDescriptor(new FileContext(this.fileSystem.FileInfo.New(outputFilePath), HttpContentType.PlainText, Encoding.UTF8.WebName, this.ExperimentId, this.AgentId, "ipmiutil_sel", this.Scenario, null, this.Roles?.FirstOrDefault()), this.Parameters, timestamped: false); + + await this.RequestFileUploadAsync(descriptor); + } + } + + /// + /// Parses the decoded SEL output from Sherlock and logs each entry as a system event. + /// + private int ParseAndLogDecodedSelEntries(string decodedOutput, EventContext telemetryContext) + { + int decodedSelCount = 0; + + if (string.IsNullOrWhiteSpace(decodedOutput)) + { + return decodedSelCount; + } + + string[] lines = decodedOutput.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + int headerIndex = -1; + + // Find the header line + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].StartsWith("ID") && lines[i].Contains("Timestamp") && lines[i].Contains("Raw Hex")) + { + headerIndex = i; + break; + } + } + + if (headerIndex == -1 || headerIndex + 2 >= lines.Length) + { + return decodedSelCount; + } + + // Skip header and separator line + int dataStartIndex = headerIndex + 2; + + for (int i = dataStartIndex; i < lines.Length; i++) + { + string line = lines[i]; + + // Skip empty lines + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + // Check if this is a new entry (starts with 0x) + if (line.TrimStart().StartsWith("0x")) + { + try + { + // Parse the main entry line + string[] parts = Regex.Split(line, @"\s{2,}"); + + if (parts.Length >= 9) + { + string id = parts[0].Trim(); + string timestampUtc = parts[1].Trim(); + string sensorOwner = parts[2].Trim(); + string sensorType = parts[3].Trim(); + string selEventType = parts[4].Trim(); + string message = parts[5].Trim(); + string selDirection = parts[6].Trim(); + string selRecordType = parts[7].Trim(); + string rawHex = parts[8].Trim(); + + // Collect continuation lines (multi-line messages) + int j = i + 1; + while (j < lines.Length && !lines[j].TrimStart().StartsWith("0x") && !lines[j].Contains("Decode Errors:") && !lines[j].Contains("Parse Errors:")) + { + string continuationLine = lines[j].Trim(); + if (!string.IsNullOrWhiteSpace(continuationLine)) + { + message = $"{message} {continuationLine}"; + } + + j++; + } + + // Update loop counter to skip processed continuation lines + i = j - 1; + + decodedSelCount++; + + // Log the parsed entry + this.Logger.LogSystemEvent( + "SelDecoded", + "SherlockIpmiUtilSelDecoder", + id, + LogLevel.Information, + telemetryContext, + null, + message, + new Dictionary + { + ["eventProvider"] = sensorOwner, + ["eventMessage"] = rawHex, + ["eventError"] = null, + ["eventResult"] = message, + ["severityLevel"] = null, + ["eventTimeStamp"] = timestampUtc, + ["sensorType"] = sensorType, + ["selEventType"] = selEventType, + ["selDirection"] = selDirection, + ["selRecordType"] = selRecordType + }); + } + } + catch (Exception ex) + { + this.Logger.LogMessage( + $"{this.TypeName}.SelEntryParsingError", + LogLevel.Warning, + telemetryContext.Clone() + .AddContext("line", line) + .AddError(ex)); + } + } + } + + return decodedSelCount; + } + + /// + /// Parses the decoded SEL output from the generic decoder and logs each entry as a system event. + /// The output has variable pipe-delimited fields: 7 for SER records, 5 for OEM timestamped, 4 for OEM non-timestamped. + /// + private int ParseAndLogGenericDecodedSelEntries(string decodedOutput, EventContext telemetryContext) + { + int decodedSelCount = 0; + + if (string.IsNullOrWhiteSpace(decodedOutput)) + { + return decodedSelCount; + } + + string[] lines = decodedOutput.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + + foreach (string line in lines) + { + string trimmedLine = line.Trim(); + if (string.IsNullOrWhiteSpace(trimmedLine)) + { + continue; + } + + try + { + string[] parts = trimmedLine.Split('|'); + + if (parts.Length >= 7) + { + // SER Record: RecordID | Timestamp | Source | SensorType | SensorID | Message | Direction + string recordId = parts[0].Trim(); + string timestampUtc = parts[1].Trim(); + string eventSource = parts[2].Trim(); + string sensorType = parts[3].Trim(); + string sensorId = parts[4].Trim(); + string message = string.Join("|", parts, 5, parts.Length - 6).Trim(); + string direction = parts[parts.Length - 1].Trim(); + + decodedSelCount++; + + this.Logger.LogSystemEvent( + "SelDecoded", + "GenericIpmiUtilSelDecoder", + recordId, + LogLevel.Information, + telemetryContext, + null, + message, + new Dictionary + { + ["eventProvider"] = eventSource, + ["eventMessage"] = message, + ["eventError"] = null, + ["eventResult"] = message, + ["severityLevel"] = null, + ["eventTimeStamp"] = timestampUtc, + ["sensorType"] = sensorType, + ["selEventType"] = sensorId, + ["selDirection"] = direction, + ["selRecordType"] = "SER" + }); + } + else if (parts.Length >= 5) + { + // OEM Timestamped: RecordID | Timestamp | Manufacturer | RecordName | Message + string recordId = parts[0].Trim(); + string timestampUtc = parts[1].Trim(); + string manufacturer = parts[2].Trim(); + string recordName = parts[3].Trim(); + string message = string.Join("|", parts, 4, parts.Length - 4).Trim(); + + decodedSelCount++; + + this.Logger.LogSystemEvent( + "SelDecoded", + "GenericIpmiUtilSelDecoder", + recordId, + LogLevel.Information, + telemetryContext, + null, + message, + new Dictionary + { + ["eventProvider"] = manufacturer, + ["eventMessage"] = message, + ["eventError"] = null, + ["eventResult"] = message, + ["severityLevel"] = null, + ["eventTimeStamp"] = timestampUtc, + ["sensorType"] = recordName, + ["selEventType"] = null, + ["selDirection"] = null, + ["selRecordType"] = "OEM_Timestamped" + }); + } + else if (parts.Length == 4) + { + // OEM Non-timestamped: RecordID | Manufacturer | RecordName | Message + string recordId = parts[0].Trim(); + string manufacturer = parts[1].Trim(); + string recordName = parts[2].Trim(); + string message = parts[3].Trim(); + + decodedSelCount++; + + this.Logger.LogSystemEvent( + "SelDecoded", + "GenericIpmiUtilSelDecoder", + recordId, + LogLevel.Information, + telemetryContext, + null, + message, + new Dictionary + { + ["eventProvider"] = manufacturer, + ["eventMessage"] = message, + ["eventError"] = null, + ["eventResult"] = message, + ["severityLevel"] = null, + ["eventTimeStamp"] = null, + ["sensorType"] = recordName, + ["selEventType"] = null, + ["selDirection"] = null, + ["selRecordType"] = "OEM_NonTimestamped" + }); + } + else + { + // Fallback: fewer than 4 pipe-delimited fields. Surface the whole line + // in eventMessage and eventResult so nothing is lost. + string recordId = parts.Length > 0 ? parts[0].Trim() : string.Empty; + + decodedSelCount++; + + this.Logger.LogSystemEvent( + "SelDecoded", + "GenericIpmiUtilSelDecoder", + recordId, + LogLevel.Information, + telemetryContext, + null, + trimmedLine, + new Dictionary + { + ["eventProvider"] = null, + ["eventMessage"] = trimmedLine, + ["eventError"] = null, + ["eventResult"] = trimmedLine, + ["severityLevel"] = null, + ["eventTimeStamp"] = null, + ["sensorType"] = null, + ["selEventType"] = null, + ["selDirection"] = null, + ["selRecordType"] = "Unknown" + }); + } + } + catch (Exception ex) + { + this.Logger.LogMessage( + $"{this.TypeName}.GenericSelEntryParsingError", + LogLevel.Warning, + telemetryContext.Clone() + .AddContext("line", line) + .AddError(ex)); + } + } + + return decodedSelCount; + } + + /// + /// Represents a parsed SEL record for suspected event analysis. + /// + protected class SelRecord + { + /// + /// Gets or sets the SEL record ID. + /// + public string RecordId { get; set; } + + /// + /// Gets or sets the timestamp of the SEL record. + /// + public string Timestamp { get; set; } + + /// + /// Gets or sets the severity level of the SEL record. + /// + public string Severity { get; set; } + + /// + /// Gets or sets the source of the SEL record. + /// + public string Source { get; set; } + + /// + /// Gets or sets the event type of the SEL record. + /// + public string EventType { get; set; } + + /// + /// Gets or sets the sensor that generated the SEL record. + /// + public string Sensor { get; set; } + + /// + /// Gets or sets the event detail description. + /// + public string EventDetail { get; set; } + + /// + /// Gets or sets a value indicating whether this is a hardware error. + /// + public bool IsHardwareError { get; set; } + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMetricsParser.cs b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMetricsParser.cs new file mode 100644 index 0000000000..1253fe2723 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMetricsParser.cs @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text.RegularExpressions; + using global::VirtualClient; + using global::VirtualClient.Contracts; + + /// + /// Parser for ipmiutil sensor output + /// + public class IpmiUtilSensorMetricsParser + { + // e.g. + // 0022 | Full | Temperature | 22 | HSC0_TMP | OK | 29.00 C + // 0023 | Full | Power Supply | 23 | HSC1_INPUT_PWR | OK | 594.00 W + // 0020 | Full | Voltage | 20 | HSC0_INPUT_VOLT | OK | 51.03 V + // Also captures non-numerical readings like: NA, N/A, ERR, ---, etc. + internal static readonly Regex SelRecordPattern = new Regex( + @"\s*(?[0-9a-z]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^|]+)\s*\|\s*(?[^\s|]+)\s*(?[^\s|]*)", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + /// + /// Gets the list of invalid IPMI sensor entries that could not be parsed as metrics + /// + public IList> InvalidIpmiSensorEntries { get; private set; } + + /// + public IList Parse(string results) + { + IList metrics = new List(); + this.InvalidIpmiSensorEntries = new List>(); + + string[] lines = Regex.Split(results, @"\r?\n"); + if (lines?.Any() == true) + { + foreach (string line in lines) + { + Match recordMatch = IpmiUtilSensorMetricsParser.SelRecordPattern.Match(line); + if (recordMatch.Success) + { + string recordId = recordMatch.Groups[1].Value?.Trim(); + string sdrType = recordMatch.Groups[2].Value?.Trim(); + string sensorType = recordMatch.Groups[3].Value?.Trim(); + string sNum = recordMatch.Groups[4].Value?.Trim(); + string sensorName = recordMatch.Groups[5].Value?.Trim(); + string sensorStatus = recordMatch.Groups[6].Value?.Trim(); + string sensorReading = recordMatch.Groups[7].Value?.Trim(); + string sensorReadingUnit = recordMatch.Groups[8].Value?.Trim(); + + // Skip header lines that contain literal field names like "ID", "Name", "Reading" + bool hasHeaderFieldNames = string.Equals(recordId, "ID", StringComparison.OrdinalIgnoreCase) + || string.Equals(sensorName, "Name", StringComparison.OrdinalIgnoreCase) + || string.Equals(sensorStatus, "Status", StringComparison.OrdinalIgnoreCase) + || string.Equals(sensorReading, "Reading", StringComparison.OrdinalIgnoreCase); + + if (hasHeaderFieldNames) + { + continue; + } + + if (!string.IsNullOrWhiteSpace(sensorType) + && !string.IsNullOrWhiteSpace(sensorName) + && !string.IsNullOrWhiteSpace(sensorReading) + && !string.IsNullOrWhiteSpace(sensorReadingUnit) + && double.TryParse(sensorReading, out double metricValue)) + { + string metricUnit = IpmiUtilSensorMetricsParser.GetMetricUnit(sensorReadingUnit, out MetricRelativity metricRelativity); + + var metadata = new Dictionary + { + ["id"] = recordId, + ["sdrType"] = sdrType, + ["type"] = sensorType, + ["snum"] = sNum, + ["name"] = sensorName, + ["status"] = sensorStatus, + ["reading"] = sensorReading, + ["readingUnit"] = sensorReadingUnit + }; + + var tags = new List() { sensorStatus }; + + metrics.Add(new Metric( + name: sensorName, + value: metricValue, + unit: metricUnit, + relativity: metricRelativity, + tags: tags, + description: line.Trim(), + metadata: metadata)); + } + else if (!string.IsNullOrWhiteSpace(sensorType) + && !string.IsNullOrWhiteSpace(sensorName) + && !double.TryParse(sensorReading, out _)) + { + // Capture non-numerical sensor readings as events + var eventInfo = new Dictionary + { + ["id"] = recordId, + ["sdrType"] = sdrType, + ["type"] = sensorType, + ["snum"] = sNum, + ["name"] = sensorName, + ["status"] = sensorStatus, + ["reading"] = sensorReading, + ["readingUnit"] = sensorReadingUnit, + ["rawLine"] = line.Trim() + }; + + this.InvalidIpmiSensorEntries.Add(eventInfo); + } + } + } + } + + return metrics; + } + + private static string GetMetricUnit(string sensorReadingUnit, out MetricRelativity relativity) + { + string unit = null; + relativity = MetricRelativity.Undefined; + + if (!string.IsNullOrWhiteSpace(sensorReadingUnit)) + { + switch (sensorReadingUnit.ToLowerInvariant()) + { + case "%": + unit = "percentage"; + break; + + case "a": + unit = MetricUnit.Amps; + break; + + case "c": + unit = MetricUnit.Celcius; + relativity = MetricRelativity.LowerIsBetter; + break; + + case "v": + unit = "volts"; + break; + + case "w": + unit = MetricUnit.Watts; + relativity = MetricRelativity.LowerIsBetter; + break; + + default: + unit = sensorReadingUnit.ToLowerInvariant(); + break; + } + } + + return unit; + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMonitor.cs b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMonitor.cs new file mode 100644 index 0000000000..1f5826e7ee --- /dev/null +++ b/src/VirtualClient/VirtualClient.Monitors/Ipmiutil/IpmiUtilSensorMonitor.cs @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Monitors +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using global::VirtualClient; + using global::VirtualClient.Common; + using global::VirtualClient.Common.Extensions; + using global::VirtualClient.Common.Telemetry; + using global::VirtualClient.Contracts; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + + /// + /// Executes the ipmiutil toolset to monitor BMC sensors on the system. + /// + [SupportedPlatforms("linux-x64,linux-arm64,win-x64,win-arm64")] + public class IpmiUtilSensorMonitor : VirtualClientIntervalBasedMonitor + { + private IFileSystem fileSystem; + private ProcessManager processManager; + private ISystemManagement systemManager; + + /// + /// Initializes a new instance of the class. + /// + public IpmiUtilSensorMonitor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.fileSystem = dependencies.GetService(); + this.processManager = dependencies.GetService(); + this.systemManager = dependencies.GetService(); + } + + /// + /// The path to the impiutil + /// + protected string IpmiUtilExePath { get; set; } + + /// + /// Initializes the monitor. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + string executableName = this.Platform == PlatformID.Unix ? "ipmiutil" : "ipmiutil.exe"; + string helpSwitch = this.Platform == PlatformID.Unix ? "--help" : "/?"; + + DependencyPath monitorsPackage = null; + + if (this.Parameters.TryGetValue(nameof(this.PackageName), out IConvertible packageName) + && !string.IsNullOrWhiteSpace(packageName?.ToString())) + { + try + { + DependencyPath toolsetPackage = await this.GetPlatformSpecificPackageAsync(packageName?.ToString(), cancellationToken, throwIfNotfound: false); + + // Account for shortcomings in the GetPlatformSpecificPackageAsync method + // implementation when throwIfNotFound = false until fixed in open source. + if (toolsetPackage != null && this.fileSystem.File.Exists(this.Combine(toolsetPackage.Path, executableName))) + { + monitorsPackage = toolsetPackage; + } + } + catch + { + // Account for bug in GetPlatformSpecificPackageAsync method when throwIfNotFound = false + // until the bug is addressed. + } + } + + if (monitorsPackage != null) + { + // e.g. + // C:\Users\AnyUser\VirtualClient\packages\host.monitors.1.0.0\win-x64\ipmiutil.exe + // /home/anyuser/VirtualClient/packages/host.monitors.1.0.0/linux-x64/ipmiutil + this.IpmiUtilExePath = this.Combine(monitorsPackage.Path, executableName); + + if (this.Platform == PlatformID.Unix) + { + await this.systemManager.MakeFileExecutableAsync(this.IpmiUtilExePath, this.Platform, cancellationToken); + } + } + else + { + // If we cannot find the ipmiutil toolset in the alternate locations, it might be available + // in a location defined in the PATH environment variable. + // + // e.g. + // ipmiutil --help + // ipmiutil /? + using (IProcessProxy process = this.processManager.CreateProcess(executableName, helpSwitch, null)) + { + await process.StartAndWaitAsync(cancellationToken); + + // ipmiutil returns various exit codes on success when using the help + // switch. We check for the presence of standard output to confirm the toolset exists. + if (!cancellationToken.IsCancellationRequested && !string.IsNullOrWhiteSpace(process.StandardOutput?.ToString())) + { + this.IpmiUtilExePath = executableName; + } + } + + // If we cannot find the ipmiutil toolset in any other location, fallback to any defined + // alternate locations. + if (string.IsNullOrWhiteSpace(this.IpmiUtilExePath)) + { + foreach (var alternateLocation in IpmiUtilSelMonitor.ToolsetAlternateLocations.Where(entry => entry.Value == this.Platform)) + { + string executablePath = alternateLocation.Key; + if (this.fileSystem.File.Exists(executablePath)) + { + this.IpmiUtilExePath = executablePath; + break; + } + } + } + } + } + catch (Exception exc) + { + // Do not let the monitor crash the application. + this.Logger.LogMessage( + $"{this.TypeName}.InitializationError", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + } + + /// + /// Monitors Ipmiutil on the target machine + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return Task.Run(async () => + { + // All background monitor ExecuteAsync methods should be either 'async' or should use a Task.Run() if running a 'while' loop or the + // logic will block without returning. Monitors are typically expected to be fire-and-forget. + + if (string.IsNullOrWhiteSpace(this.IpmiUtilExePath)) + { + this.Logger.LogMessage( + $"The 'ipmiutil' toolset was not found on the system. This monitor will exit.", + LogLevel.Warning, + telemetryContext); + } + else + { + long currentIteration = 1; + DateTime nextIteration = DateTime.UtcNow.Add(this.MonitorWarmupPeriod); + while (!cancellationToken.IsCancellationRequested && !this.IsIterationComplete(currentIteration)) + { + try + { + await this.WaitAsync(nextIteration, cancellationToken); + nextIteration = DateTime.UtcNow.Add(this.MonitorFrequency); + + if (cancellationToken.IsCancellationRequested) + { + break; + } + + string command = this.IpmiUtilExePath; + string commandArguments = "sensor -s"; + + telemetryContext.AddContext("command", command); + telemetryContext.AddContext("commandArguments", commandArguments); + + using (IProcessProxy process = await this.ExecuteCommandAsync(command, commandArguments, null, telemetryContext, CancellationToken.None, runElevated: true)) + { + await this.LogProcessDetailsAsync(process, telemetryContext, toolName: "ipmiutil_sensor", logFileName: "ipmiutil_sensor"); + process.ThrowIfMonitorFailed(); + + if (process.ExitCode == 0) + { + IpmiUtilSensorMetricsParser parser = new IpmiUtilSensorMetricsParser(); + IList metrics = parser.Parse(process.StandardOutput.ToString()); + + if (metrics?.Any() == true) + { + this.Logger.LogMetrics( + toolName: "ipmiutil", + scenarioName: "Sensors", + process.StartTime, + process.ExitTime, + metrics, + metricCategorization: "IPMI", + scenarioArguments: process.FullCommand(), + this.Tags, + telemetryContext); + } + + if (parser.InvalidIpmiSensorEntries?.Any() == true) + { + foreach (var sensorEvent in parser.InvalidIpmiSensorEntries) + { + if (sensorEvent.TryGetValue("name", out var name)) + { + string sensorName = name?.ToString(); + + EventContext relatedContext = telemetryContext.Clone(); + relatedContext.AddContext("sensorName", sensorName); + + this.Logger.LogSystemEvent( + "ipmi_sensor_event", + "ipmiutil", + sensorName, + LogLevel.Warning, + relatedContext, + eventDescription: $"{this.IpmiUtilExePath} {commandArguments}", + eventInfo: sensorEvent); + } + } + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.{this.Scenario}Warning", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + finally + { + currentIteration++; + } + } + } + }); + } + } +} \ No newline at end of file diff --git a/website/docs/monitors/ipmiutil.md b/website/docs/monitors/ipmiutil.md new file mode 100644 index 0000000000..f66f6cc72c --- /dev/null +++ b/website/docs/monitors/ipmiutil.md @@ -0,0 +1,120 @@ +--- +slug: /monitors/ipmiutil +sidebar_position: 7 +--- + +# IPMIUtil Monitor + +IPMIUtil is a collection of utilities for systems that implement the Intelligent Platform Management Interface (IPMI). Virtual Client uses IPMIUtil as a background monitor to capture sensor readings and Baseboard Management Controller (BMC) System Event Log (SEL) records while a workload runs. + +* [IPMIUtil project](https://ipmiutil.sourceforge.net/) +* [IPMIUtil user guide](https://ipmiutil.sourceforge.net/docs/UserGuide) +* [IPMIUtil source](https://sourceforge.net/projects/ipmiutil/) + +## Supported Platforms + +* linux-x64 +* linux-arm64 +* win-x64 +* win-arm64 + +The system must expose an IPMI/BMC interface that IPMIUtil can access. The monitor does not emulate BMC hardware and cannot produce data on systems without supported management hardware and drivers. + +## Installation + +The `MONITORS-IPMIUTIL.json` profile installs IPMIUtil from the operating-system package repository on Linux. On Windows, it downloads `host.monitors.1.0.1.zip` from the Virtual Client `packages` container, so a package store must be supplied when the package is not already cached. + +``` bash +# Linux +sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 --logger=csv --log-to-file + +# Windows +VirtualClient.exe --profile=MONITORS-IPMIUTIL.json --timeout=30 --packages="" --logger=csv --log-to-file +``` + +The monitors also search for `ipmiutil` or `ipmiutil.exe` on `PATH`. On Windows, the legacy `C:\BladeFX_latest\BladeFX\Tools\IpmiUtil\ipmiutil.exe` location is checked as a final fallback. + +## Sensor Monitoring + +`IpmiUtilSensorMonitor` executes: + +``` text +ipmiutil sensor -s +``` + +Each numeric sensor reading is emitted as a metric. The metric name is the IPMI sensor name, and the sensor status is included as a tag. The parser maps common units as follows: + +| IPMI unit | Virtual Client unit | Relativity | +|-----------|---------------------|------------| +| `A` | amps | Undefined | +| `C` | Celsius | Lower is better | +| `V` | volts | Undefined | +| `W` | watts | Lower is better | +| `%` | percentage | Undefined | + +Non-numeric readings such as `N/A`, `ERR`, or `---` are emitted as monitor events rather than discarded. + +## SEL Monitoring + +`IpmiUtilSelMonitor` uses the following commands: + +| Command | Purpose | +|---------|---------| +| `ipmiutil sel -uc` | Read decoded SEL records. | +| `ipmiutil sel -r` | Read raw SEL records and combine them with decoded records. | +| `ipmiutil sel -d` | Clear the SEL when `ClearSel` is enabled. | + +Matching SEL records are emitted as count metrics. Metric metadata preserves the record ID, severity, sensor, timestamp, source, event type, decoded details, and raw details. Tags include `IPMI`, the event source, and severity. + +Common metric names include: + +| SEL event | Example metric | +|-----------|----------------| +| Processor internal error | `Processor_IERR` | +| Processor FRB2 timeout | `Processor_FRB2` | +| Uncorrectable memory ECC | `Memory_Uncorrectable_ECC` | +| Correctable memory ECC | `Memory_Correctable_ECC` | +| Fatal non-maskable interrupt | `Critical_Interrupt_Fatal_NMI` | +| Power-supply predictive failure | `Power_Supply_Predictive_Failure` | + +## Parameters + +The following parameters can be overridden on the command line. + +| Parameter | Purpose | Default | +|-----------|---------|---------| +| `MonitorFrequency` | Interval between captures. | `00:01:00` | +| `MonitorWarmupPeriod` | Delay before the first capture. | `00:00:00` | +| `MonitorIterations` | Maximum captures; `-1` continues until cancellation. | `-1` | +| `ClearSel` | Clears the BMC SEL after it is captured. Use with care because this modifies persistent BMC state. | `false` | +| `EventsFilter` | Regular expression applied to decoded SEL records. | `MAJ\|CRT` | +| `CaptureSuspectedEvents` | Writes suspected hardware events to separate output files. | `false` | +| `IncludeWarningsInSuspectedEvents` | Includes warning-level records when suspected-event capture is enabled. | `true` | +| `EnableSelDecoding` | Runs an optional Sherlock or Generic decoder against raw SEL data. | `false` | +| `DecoderType` | Decoder engine: `Sherlock` or `Generic`. Required when decoding is enabled. | | +| `DecoderArguments` | Additional arguments passed to the selected decoder. | | +| `DecoderPackageName` | Installed package containing the decoder executable. | | +| `DecoderExecutableName` | Overrides the decoder executable name. | | + +Optional SEL decoding requires a separately supplied decoder package. The IPMIUtil package does not include either decoder. + +## Usage Examples + +``` bash +# Capture sensors and major/critical SEL events for 30 minutes. +sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 + +# Capture all decoded SEL severities without clearing the SEL. +sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 --parameters="EventsFilter=.*;;;ClearSel=false" + +# Capture only one iteration after a 10-second warmup. +VirtualClient.exe --profile=MONITORS-IPMIUTIL.json --timeout=5 --parameters="MonitorIterations=1;;;MonitorWarmupPeriod=00:00:10" --packages="" +``` + +## Troubleshooting + +* Run Virtual Client with elevated privileges when the operating system restricts access to the IPMI device. +* Confirm `ipmiutil --help` on Linux or `ipmiutil.exe /?` on Windows returns output before diagnosing the monitor. +* An empty result can mean the system has no accessible BMC sensors or SEL records; it does not necessarily indicate a parser failure. +* If the Windows dependency fails, confirm the public package store contains `packages/host.monitors.1.0.1.zip`. +* Avoid enabling `ClearSel` during diagnosis until the SEL data has been preserved. diff --git a/website/docs/monitors/monitor-profiles.md b/website/docs/monitors/monitor-profiles.md index dd8f8b40fa..17045baa7e 100644 --- a/website/docs/monitors/monitor-profiles.md +++ b/website/docs/monitors/monitor-profiles.md @@ -8,6 +8,13 @@ The following sections describe the various monitor profiles that are available define the background monitors that will run on the system. Monitors are often ran in conjunction with workloads (defined in workload profiles) in order to capture performance and reliability information from the system while workloads are running. +## MONITORS-AZURE-HOST.json +This compatibility profile captures IPMIUtil sensors, BMC SEL records, and Windows performance counters on an Azure host or blade. It preserves the monitor names, intervals, event filter, and counter definitions from the profile previously distributed by the internal Virtual Client extension. + +The profile supports `win-x64` and `win-arm64` and requires direct access to an IPMI-capable physical host. IPMIUtil must already be available on `PATH`, in the legacy BladeFX tool location, or in an installed `host.monitors` package. + +See the [IPMIUtil monitor documentation](/docs/monitors/ipmiutil) for hardware requirements and emitted telemetry. + ## MONITORS-DEFAULT.json The default monitor profile for the Virtual Client. This profile captures performance counters on the system using one or more different specialized toolsets. This monitor profile will be used when no other monitor profiles are specified on the command line. @@ -140,3 +147,77 @@ any of the dependencies required by the driver. If the driver is not already ins # Run the monitoring facilities only. ./VirtualClient --profile=MONITORS-GPU-NVIDIA.json --logger=csv --log-to-file ``` + +## MONITORS-ETW.json +This profile runs both the kernel and non-kernel [Windows ETW monitor](/docs/monitors/windows-etw) configurations. + +* **Supported Platform-Architectures** + * win-x64 + * win-arm64 + +* **Scenarios** + * `CaptureKernelETWTraces` captures process start, process stop, and image-load events from the Windows kernel provider. + * `CaptureNonKernelETWTraces` captures selected garbage-collection events from the .NET runtime provider. + +* **Profile Parameters** + + | Parameter | Purpose | Default value | + |-----------|---------|---------------| + | ProfilingEnabled | Enables or disables ETW collection. | `true` | + | ProfilingMode | Collection mode (`Interval` or `OnDemand`). | `Interval` | + | ProfilingPeriod | Length of each interval collection period. | `00:05:00` | + | ProfilingInterval | Delay between interval collection periods. | `00:00:30` | + | ProfilingWarmUpPeriod | Delay before interval collection starts. | `00:00:10` | + +* **Usage Examples** + + ```powershell + # Run both ETW monitor configurations. + VirtualClient.exe --profile=MONITORS-ETW.json --timeout=00:30:00 --log-to-file + + # Run ETW monitoring with a workload profile. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW.json --timeout=00:30:00 --log-to-file + ``` + +## MONITORS-ETW-KERNEL.json +This profile runs the [Windows ETW monitor](/docs/monitors/windows-etw) for the Windows kernel provider. It captures `ImageLoad`, +`ProcessStart`, and `ProcessStop` events enabled by the `ImageLoad` and `Process` keywords. + +The profile supports win-x64 and win-arm64 and exposes the same profile parameters documented for +[MONITORS-ETW.json](#monitors-etwjson). + +```powershell +VirtualClient.exe --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --log-to-file +``` + +## MONITORS-ETW-NONKERNEL.json +This profile runs the [Windows ETW monitor](/docs/monitors/windows-etw) for the `Microsoft-Windows-DotNETRuntime` provider. It captures +`GCSetGCHandle` and `GCTriggered` events at the configured levels. + +The profile supports win-x64 and win-arm64 and exposes the same profile parameters documented for +[MONITORS-ETW.json](#monitors-etwjson). + +```powershell +VirtualClient.exe --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --log-to-file +``` + +## MONITORS-IPMIUTIL.json +The IPMIUtil monitor profile captures BMC sensor measurements and System Event Log records throughout a Virtual Client run. + +* **Supported Platform-Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Dependencies** + * A supported IPMI/BMC interface and sufficient privileges to access it. + * Linux access to an operating-system package repository that provides `ipmiutil`. + * Windows access to the Virtual Client package store containing `host.monitors.1.0.1.zip`. + +* **Scenarios** + * Capture numeric BMC sensor readings. + * Capture and combine decoded and raw BMC SEL records. + * Optionally preserve suspected hardware events and clear the SEL. + +See the [IPMIUtil monitor documentation](/docs/monitors/ipmiutil) for metrics, parameters, setup requirements, and examples. diff --git a/website/docs/monitors/systemevents/windows-etw.md b/website/docs/monitors/systemevents/windows-etw.md new file mode 100644 index 0000000000..bbb75c141a --- /dev/null +++ b/website/docs/monitors/systemevents/windows-etw.md @@ -0,0 +1,124 @@ +--- +slug: /monitors/windows-etw +--- + +# Windows Event Tracing +Event Tracing for Windows (ETW) is a Windows facility for collecting events from operating-system and application providers. The +`WindowsETWMonitor` starts an ETW session, enables the configured provider, filters the events when requested, and emits each matching event +through the Virtual Client system-event telemetry channel. + +* [Event Tracing for Windows](https://learn.microsoft.com/windows/win32/etw/event-tracing-portal) +* [About Event Tracing](https://learn.microsoft.com/windows/win32/etw/about-event-tracing) + +## Supported Platforms +* win-x64 +* win-arm64 + +The monitor requires Windows 8/Windows Server 2012 or later and Virtual Client must run with administrator privileges. + +## Provider Types +The monitor supports the following provider types: + +* **Kernel** - Enables Windows kernel providers. `ProviderKeywords` is required and `ProviderEvents` can limit collection to specific kernel + events. +* **NonKernel** - Enables an application or system provider by name or GUID. The provider's dynamic events are collected. + +Only one provider is configured per monitor instance. Add multiple monitor entries to a profile to collect from multiple providers. + +## Output +Each matching ETW event is emitted as a Virtual Client system event with: + +| Field | Value | +|-------|-------| +| Event type | `ETW` | +| Event source | `Windows ETW` | +| Event ID | The ETW event ID, when available | +| Event description | The ETW event name, when available | +| Severity | `Information` | +| Event information | ETW header fields and provider-specific payload fields | + +The event information includes standard ETW fields such as `ActivityID`, `ProviderGuid`, `ProviderName`, `EventID`, `Version`, `Level`, +`Task`, `TaskName`, `OpCode`, `OpCodeName`, `Keywords`, `TimeStamp`, `ProcessID`, `ProcessName`, `ProcessorNumber`, `ThreadID`, and +`EventName`. Provider-specific payload fields are added using their ETW payload names. + +When `--log-to-file` is specified, the configured Virtual Client logger writes these events to the local logs directory. Other configured +telemetry loggers can consume the same system-event records. + +## Parameters +The following parameters are available for the `WindowsETWMonitor` component. + +| Parameter | Required | Description | Default value | +|-----------|----------|-------------|---------------| +| ProviderType | Yes | Provider category. Supported values are `Kernel` and `NonKernel`. | | +| Provider | Yes | Provider name or GUID. Use `Windows Kernel` for the kernel provider. | | +| ProviderKeywords | Kernel only | Comma-delimited `KernelTraceEventParser.Keywords` values used to enable kernel events (for example, `ImageLoad,Process`). | | +| ProviderEvents | No | Comma-delimited kernel event names to capture (for example, `ImageLoad,ProcessStart,ProcessStop`). When omitted for a kernel provider, all events enabled by the keywords are captured. | | +| JsonFilters | No | JSON object whose values are string arrays. An event must match every filter key. Key and value comparisons are case-insensitive (for example, `{'EventName':['ProcessStart'],'Level':['4']}`). | | +| ProfilingEnabled | No | Enables or disables collection. | `true` | +| ProfilingMode | No | Collection mode. Supported values are `Interval` and `OnDemand`. | `None` | +| ProfilingPeriod | Interval mode | Length of each collection period. | `00:05:00` | +| ProfilingInterval | Interval mode | Delay between collection periods. | `00:00:30` | +| ProfilingWarmUpPeriod | No | Delay before interval collection begins. On-demand requests supply their own warm-up period. | `00:00:10` in the provided profiles | +| Scenario | No | Name describing the purpose of the monitor in the profile. | | + +`JsonFilters` values must be strings, including values that represent numbers. For example: + +```json +"JsonFilters": "{'ProcessName':['cmd'],'EventName':['ProcessStart'],'ExitStatus':['259']}" +``` + +## Usage Examples +The following example captures process and image-load kernel events on an interval: + +```json +"Monitors": [ + { + "Type": "WindowsETWMonitor", + "Parameters": { + "Scenario": "CaptureKernelETWTraces", + "ProfilingEnabled": true, + "ProfilingMode": "Interval", + "ProfilingPeriod": "00:05:00", + "ProfilingInterval": "00:00:30", + "ProfilingWarmUpPeriod": "00:00:10", + "ProviderType": "Kernel", + "Provider": "Windows Kernel", + "ProviderKeywords": "ImageLoad,Process", + "ProviderEvents": "ImageLoad,ProcessStart,ProcessStop", + "JsonFilters": "" + } + } +] +``` + +The following example captures selected .NET runtime events: + +```json +"Monitors": [ + { + "Type": "WindowsETWMonitor", + "Parameters": { + "Scenario": "CaptureNonKernelETWTraces", + "ProfilingEnabled": true, + "ProfilingMode": "Interval", + "ProfilingPeriod": "00:05:00", + "ProfilingInterval": "00:00:30", + "ProfilingWarmUpPeriod": "00:00:10", + "ProviderType": "NonKernel", + "Provider": "Microsoft-Windows-DotNETRuntime", + "JsonFilters": "{'EventName':['GCSetGCHandle','GCTriggered'],'Level':['4','2']}" + } + } +] +``` + +Virtual Client includes ready-to-use profiles for kernel, non-kernel, and combined collection. See +[Monitor Profiles](/docs/monitors/monitor-profiles#monitors-etwjson). + +## Troubleshooting +* **Unauthorized error** - Start Virtual Client from an elevated command prompt or PowerShell session. +* **No events captured** - Confirm the provider name or GUID, keywords, and event names. The provider must emit events while the session is + active. +* **No events after adding filters** - Confirm each filter key matches an emitted header or payload field and each value is represented as a + string in an array. +* **Platform-not-supported error** - Run the monitor on Windows 8/Windows Server 2012 or later. diff --git a/website/docs/overview/overview.md b/website/docs/overview/overview.md index d71d4fba38..be5447e274 100644 --- a/website/docs/overview/overview.md +++ b/website/docs/overview/overview.md @@ -92,6 +92,7 @@ for using them. | [Gzip](/docs/workloads/compression/gzip) | Compression | linux-x64, linux-arm64 | [GPL](https://www.gnu.org/software/gzip/) | | [HPCG](/docs/workloads/hpcg) | High Performance Compute (HPC) | linux-x64, linux-arm64 | [Custom](https://github.com/hpcg-benchmark/hpcg/blob/master/COPYING) | | [HPLinpack](/docs/workloads/hplinpack) | Linear Equations | linux-x64, linux-arm64| [IBM](https://netlib.org/benchmark/hpl/IBM_LICENSE.TXT) | +| [IPMIUtil](/docs/monitors/ipmiutil) | BMC Sensor and System Event Monitoring | linux-x64, linux-arm64, win-x64, win-arm64 | [BSD-2-Clause](https://sourceforge.net/p/ipmiutil/code/ci/master/tree/COPYING) | | [LAPACK](/docs/workloads/lapack) | Linear Equations | linux-x64, linux-arm64, win-x64, win-arm64 | [Custom](https://github.com/Reference-LAPACK/lapack/blob/master/LICENSE) | | [Latte](/docs/workloads/network-suite) | Network Latencies | win-x64, win-arm64 | [MIT](https://github.com/microsoft/latte/blob/main/LICENSE) | | [LMbench](/docs/workloads/lmbench) | Memory Performance | linux-x64, linux-arm64 | [GPL-2.0](https://github.com/intel/lmbench/blob/master/COPYING) | From f6d78b7a6a0840d2e9b97d90e06f70ae92f92444 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Fri, 9 Oct 2026 15:53:04 -0700 Subject: [PATCH 2/7] Correct migrated monitor documentation Align the public documentation with the parity-preserved behavior of the migrated implementations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- website/docs/monitors/ipmiutil.md | 3 ++- website/docs/monitors/monitor-profiles.md | 2 +- website/docs/monitors/systemevents/windows-etw.md | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/website/docs/monitors/ipmiutil.md b/website/docs/monitors/ipmiutil.md index f66f6cc72c..ed51788d72 100644 --- a/website/docs/monitors/ipmiutil.md +++ b/website/docs/monitors/ipmiutil.md @@ -32,7 +32,8 @@ sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 --logger=csv VirtualClient.exe --profile=MONITORS-IPMIUTIL.json --timeout=30 --packages="" --logger=csv --log-to-file ``` -The monitors also search for `ipmiutil` or `ipmiutil.exe` on `PATH`. On Windows, the legacy `C:\BladeFX_latest\BladeFX\Tools\IpmiUtil\ipmiutil.exe` location is checked as a final fallback. +The monitors search for `ipmiutil` or `ipmiutil.exe` on `PATH`. The SEL monitor also checks the legacy +`C:\BladeFX_latest\BladeFX\Tools\IpmiUtil\ipmiutil.exe` location as a final fallback. ## Sensor Monitoring diff --git a/website/docs/monitors/monitor-profiles.md b/website/docs/monitors/monitor-profiles.md index 17045baa7e..ce5d24fb44 100644 --- a/website/docs/monitors/monitor-profiles.md +++ b/website/docs/monitors/monitor-profiles.md @@ -11,7 +11,7 @@ order to capture performance and reliability information from the system while w ## MONITORS-AZURE-HOST.json This compatibility profile captures IPMIUtil sensors, BMC SEL records, and Windows performance counters on an Azure host or blade. It preserves the monitor names, intervals, event filter, and counter definitions from the profile previously distributed by the internal Virtual Client extension. -The profile supports `win-x64` and `win-arm64` and requires direct access to an IPMI-capable physical host. IPMIUtil must already be available on `PATH`, in the legacy BladeFX tool location, or in an installed `host.monitors` package. +The profile supports `win-x64` and `win-arm64` and requires direct access to an IPMI-capable physical host. IPMIUtil must already be available on `PATH`. The SEL monitor can also use the legacy BladeFX tool location. See the [IPMIUtil monitor documentation](/docs/monitors/ipmiutil) for hardware requirements and emitted telemetry. diff --git a/website/docs/monitors/systemevents/windows-etw.md b/website/docs/monitors/systemevents/windows-etw.md index bbb75c141a..eac5757fe1 100644 --- a/website/docs/monitors/systemevents/windows-etw.md +++ b/website/docs/monitors/systemevents/windows-etw.md @@ -21,7 +21,7 @@ The monitor supports the following provider types: * **Kernel** - Enables Windows kernel providers. `ProviderKeywords` is required and `ProviderEvents` can limit collection to specific kernel events. -* **NonKernel** - Enables an application or system provider by name or GUID. The provider's dynamic events are collected. +* **NonKernel** - Enables an application or system provider by name. The provider's dynamic events are collected. Only one provider is configured per monitor instance. Add multiple monitor entries to a profile to collect from multiple providers. @@ -50,7 +50,7 @@ The following parameters are available for the `WindowsETWMonitor` component. | Parameter | Required | Description | Default value | |-----------|----------|-------------|---------------| | ProviderType | Yes | Provider category. Supported values are `Kernel` and `NonKernel`. | | -| Provider | Yes | Provider name or GUID. Use `Windows Kernel` for the kernel provider. | | +| Provider | Yes | Provider name. Use `Windows Kernel` for the kernel provider. | | | ProviderKeywords | Kernel only | Comma-delimited `KernelTraceEventParser.Keywords` values used to enable kernel events (for example, `ImageLoad,Process`). | | | ProviderEvents | No | Comma-delimited kernel event names to capture (for example, `ImageLoad,ProcessStart,ProcessStop`). When omitted for a kernel provider, all events enabled by the keywords are captured. | | | JsonFilters | No | JSON object whose values are string arrays. An event must match every filter key. Key and value comparisons are case-insensitive (for example, `{'EventName':['ProcessStart'],'Level':['4']}`). | | From 351e5d2321c2738c4a76e5dc65b3db36acf5da00 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Fri, 9 Oct 2026 16:06:49 -0700 Subject: [PATCH 3/7] Remove non-parity ETW profile tests Keep the migration scoped to tests that existed in the internal implementation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../WindowsETWMonitorProfileTests.cs | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs deleted file mode 100644 index d9e6d2521b..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Linq; - using NUnit.Framework; - using VirtualClient.Contracts; - using VirtualClient.Monitors; - using VirtualClient.TestExtensions; - - [TestFixture] - [Category("Functional")] - public class WindowsETWMonitorProfileTests - { - private DependencyFixture mockFixture; - - [OneTimeSetUp] - public void SetupFixture() - { - this.mockFixture = new DependencyFixture(); - ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); - } - - [Test] - [TestCase("MONITORS-ETW.json", 2)] - [TestCase("MONITORS-ETW-KERNEL.json", 1)] - [TestCase("MONITORS-ETW-NONKERNEL.json", 1)] - public void WindowsETWMonitorProfileParametersAreInlinedCorrectly(string profile, int expectedMonitorCount) - { - this.mockFixture.Setup(PlatformID.Win32NT); - - using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - WorkloadAssert.ParameterReferencesInlined(executor.Profile); - Assert.AreEqual(expectedMonitorCount, executor.Profile.Monitors.Count); - Assert.IsTrue(executor.Profile.Monitors.All(monitor => monitor.Type == nameof(WindowsETWMonitor))); - } - } - } -} From 47185bbf2cf91e8ecc00750416256908c8fbdc40 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Fri, 9 Oct 2026 16:09:52 -0700 Subject: [PATCH 4/7] Add migrated monitor profile coverage Follow public workload precedent by verifying profile loading, parameter inlining, and migrated component type wiring for ETW and IPMIUtil. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../IpmiUtilProfileTests.cs | 47 +++++++++++++++++++ .../WindowsETWMonitorProfileTests.cs | 41 ++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs new file mode 100644 index 0000000000..eb79a8d046 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Linq; + using NUnit.Framework; + using VirtualClient.Contracts; + using VirtualClient.TestExtensions; + + [TestFixture] + [Category("Functional")] + public class IpmiUtilProfileTests + { + private DependencyFixture mockFixture; + + [OneTimeSetUp] + public void SetupFixture() + { + this.mockFixture = new DependencyFixture(); + ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); + } + + [Test] + [TestCase("MONITORS-IPMIUTIL.json", PlatformID.Unix, 2, 2)] + [TestCase("MONITORS-IPMIUTIL.json", PlatformID.Win32NT, 2, 2)] + [TestCase("MONITORS-AZURE-HOST.json", PlatformID.Win32NT, 3, 0)] + public void IpmiUtilProfileParametersAreInlinedCorrectly( + string profile, + PlatformID platform, + int expectedMonitorCount, + int expectedDependencyCount) + { + this.mockFixture.Setup(platform); + + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + Assert.AreEqual(expectedMonitorCount, executor.Profile.Monitors.Count); + Assert.AreEqual(expectedDependencyCount, executor.Profile.Dependencies.Count); + Assert.IsTrue(executor.Profile.Monitors.Any(monitor => monitor.Type == "IpmiUtilSensorMonitor")); + Assert.IsTrue(executor.Profile.Monitors.Any(monitor => monitor.Type == "IpmiUtilSelMonitor")); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs new file mode 100644 index 0000000000..9b9a3e620d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Linq; + using NUnit.Framework; + using VirtualClient.Contracts; + using VirtualClient.TestExtensions; + + [TestFixture] + [Category("Functional")] + public class WindowsETWMonitorProfileTests + { + private DependencyFixture mockFixture; + + [OneTimeSetUp] + public void SetupFixture() + { + this.mockFixture = new DependencyFixture(); + ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); + } + + [Test] + [TestCase("MONITORS-ETW.json", 2)] + [TestCase("MONITORS-ETW-KERNEL.json", 1)] + [TestCase("MONITORS-ETW-NONKERNEL.json", 1)] + public void WindowsETWMonitorProfileParametersAreInlinedCorrectly(string profile, int expectedMonitorCount) + { + this.mockFixture.Setup(PlatformID.Win32NT); + + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + Assert.AreEqual(expectedMonitorCount, executor.Profile.Monitors.Count); + Assert.IsTrue(executor.Profile.Monitors.All(monitor => monitor.Type == "WindowsETWMonitor")); + } + } + } +} From 4628175010f383c0b25e53c791e0e64d4596382d Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Mon, 12 Oct 2026 10:08:44 -0700 Subject: [PATCH 5/7] Fix IPMIUtil path assertion on Linux Normalize the captured Windows path separator before extracting the filename in the migrated timestamp test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Ipmiutil/IpmiUtilSelMonitorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs index d00781447c..76d923a519 100644 --- a/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs +++ b/src/VirtualClient/VirtualClient.Monitors.UnitTests/Ipmiutil/IpmiUtilSelMonitorTests.cs @@ -1829,7 +1829,7 @@ public async Task IpmiUtilSelMonitorGenericDecodingUsesMillisecondGranularityInF Assert.IsNotNull(capturedInputPath); // Millisecond format: yyyy-MM-dd-HH-mm-ss-fff — the timestamp portion has 23 chars // Example: 2026-05-17-10-27-15-830_input_selRecords.txt - string fileName = Path.GetFileName(capturedInputPath); + string fileName = Path.GetFileName(capturedInputPath.Replace('\\', Path.DirectorySeparatorChar)); string timestampPart = fileName.Substring(0, fileName.IndexOf("_input_selRecords")); Assert.AreEqual(23, timestampPart.Length, "Timestamp should be in yyyy-MM-dd-HH-mm-ss-fff format (23 chars)."); } From 90c8c804a69bf8567e4250e7e6a73d0d4e51dba9 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Mon, 12 Oct 2026 12:32:57 -0700 Subject: [PATCH 6/7] Restore standardized ETW usage examples Preserve the internal monitor-only, workload-composed, and remote telemetry command scenarios using current public logger semantics and profile documentation conventions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- website/docs/monitors/monitor-profiles.md | 66 +++++++++++++++---- .../docs/monitors/systemevents/windows-etw.md | 2 +- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/website/docs/monitors/monitor-profiles.md b/website/docs/monitors/monitor-profiles.md index ce5d24fb44..a7fcd5efad 100644 --- a/website/docs/monitors/monitor-profiles.md +++ b/website/docs/monitors/monitor-profiles.md @@ -155,6 +155,9 @@ This profile runs both the kernel and non-kernel [Windows ETW monitor](/docs/mon * win-x64 * win-arm64 +* **Dependencies** + * Virtual Client must run with administrator privileges. + * **Scenarios** * `CaptureKernelETWTraces` captures process start, process stop, and image-load events from the Windows kernel provider. * `CaptureNonKernelETWTraces` captures selected garbage-collection events from the .NET runtime provider. @@ -172,34 +175,69 @@ This profile runs both the kernel and non-kernel [Windows ETW monitor](/docs/mon * **Usage Examples** ```powershell - # Run both ETW monitor configurations. - VirtualClient.exe --profile=MONITORS-ETW.json --timeout=00:30:00 --log-to-file + # Run both ETW monitor configurations and write telemetry locally. + VirtualClient.exe --profile=MONITORS-ETW.json --timeout=00:30:00 --logger=file - # Run ETW monitoring with a workload profile. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW.json --timeout=00:30:00 --log-to-file + # Run both ETW monitor configurations with a workload profile. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW.json --timeout=00:30:00 --logger=file + + # Upload ETW telemetry to an Event Hub. Internet connectivity is required. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW.json --timeout=00:30:00 --event-hub="" ``` ## MONITORS-ETW-KERNEL.json This profile runs the [Windows ETW monitor](/docs/monitors/windows-etw) for the Windows kernel provider. It captures `ImageLoad`, `ProcessStart`, and `ProcessStop` events enabled by the `ImageLoad` and `Process` keywords. -The profile supports win-x64 and win-arm64 and exposes the same profile parameters documented for -[MONITORS-ETW.json](#monitors-etwjson). +* **Supported Platform-Architectures** + * win-x64 + * win-arm64 -```powershell -VirtualClient.exe --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --log-to-file -``` +* **Dependencies** + * Virtual Client must run with administrator privileges. + +* **Profile Parameters** + See [MONITORS-ETW.json](#monitors-etwjson). + +* **Usage Examples** + + ```powershell + # Run the kernel ETW monitor and write telemetry locally. + VirtualClient.exe --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --logger=file + + # Run the kernel ETW monitor with a workload profile. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --logger=file + + # Upload kernel ETW telemetry to an Event Hub. Internet connectivity is required. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --event-hub="" + ``` ## MONITORS-ETW-NONKERNEL.json This profile runs the [Windows ETW monitor](/docs/monitors/windows-etw) for the `Microsoft-Windows-DotNETRuntime` provider. It captures `GCSetGCHandle` and `GCTriggered` events at the configured levels. -The profile supports win-x64 and win-arm64 and exposes the same profile parameters documented for -[MONITORS-ETW.json](#monitors-etwjson). +* **Supported Platform-Architectures** + * win-x64 + * win-arm64 + +* **Dependencies** + * Virtual Client must run with administrator privileges. + +* **Profile Parameters** + See [MONITORS-ETW.json](#monitors-etwjson). + +* **Usage Examples** -```powershell -VirtualClient.exe --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --log-to-file -``` + ```powershell + # Run the non-kernel ETW monitor and write telemetry locally. + VirtualClient.exe --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --logger=file + + # Run the non-kernel ETW monitor with a workload profile. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --logger=file + + # Upload non-kernel ETW telemetry to an Event Hub. Internet connectivity is required. + VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --event-hub="" + ``` ## MONITORS-IPMIUTIL.json The IPMIUtil monitor profile captures BMC sensor measurements and System Event Log records throughout a Virtual Client run. diff --git a/website/docs/monitors/systemevents/windows-etw.md b/website/docs/monitors/systemevents/windows-etw.md index eac5757fe1..0f0a02910f 100644 --- a/website/docs/monitors/systemevents/windows-etw.md +++ b/website/docs/monitors/systemevents/windows-etw.md @@ -117,7 +117,7 @@ Virtual Client includes ready-to-use profiles for kernel, non-kernel, and combin ## Troubleshooting * **Unauthorized error** - Start Virtual Client from an elevated command prompt or PowerShell session. -* **No events captured** - Confirm the provider name or GUID, keywords, and event names. The provider must emit events while the session is +* **No events captured** - Confirm the provider name, keywords, and event names. The provider must emit events while the session is active. * **No events after adding filters** - Confirm each filter key matches an emitted header or payload field and each value is represented as a string in an array. From f85b7f684349ec2e0112a0880804bd69737cee99 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Mon, 12 Oct 2026 14:29:16 -0700 Subject: [PATCH 7/7] Remove bundled monitor profiles Keep ETW and IPMIUtil usage in component documentation without adding public monitor profiles. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../IpmiUtilProfileTests.cs | 47 ------- .../WindowsETWMonitorProfileTests.cs | 41 ------ .../profiles/MONITORS-AZURE-HOST.json | 51 -------- .../profiles/MONITORS-ETW-KERNEL.json | 32 ----- .../profiles/MONITORS-ETW-NONKERNEL.json | 30 ----- .../profiles/MONITORS-ETW.json | 47 ------- .../profiles/MONITORS-IPMIUTIL.json | 76 ----------- website/docs/monitors/ipmiutil.md | 81 +++++++++--- website/docs/monitors/monitor-profiles.md | 119 ------------------ .../docs/monitors/systemevents/windows-etw.md | 23 +++- 10 files changed, 84 insertions(+), 463 deletions(-) delete mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs deleted file mode 100644 index eb79a8d046..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/IpmiUtilProfileTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Linq; - using NUnit.Framework; - using VirtualClient.Contracts; - using VirtualClient.TestExtensions; - - [TestFixture] - [Category("Functional")] - public class IpmiUtilProfileTests - { - private DependencyFixture mockFixture; - - [OneTimeSetUp] - public void SetupFixture() - { - this.mockFixture = new DependencyFixture(); - ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); - } - - [Test] - [TestCase("MONITORS-IPMIUTIL.json", PlatformID.Unix, 2, 2)] - [TestCase("MONITORS-IPMIUTIL.json", PlatformID.Win32NT, 2, 2)] - [TestCase("MONITORS-AZURE-HOST.json", PlatformID.Win32NT, 3, 0)] - public void IpmiUtilProfileParametersAreInlinedCorrectly( - string profile, - PlatformID platform, - int expectedMonitorCount, - int expectedDependencyCount) - { - this.mockFixture.Setup(platform); - - using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - WorkloadAssert.ParameterReferencesInlined(executor.Profile); - Assert.AreEqual(expectedMonitorCount, executor.Profile.Monitors.Count); - Assert.AreEqual(expectedDependencyCount, executor.Profile.Dependencies.Count); - Assert.IsTrue(executor.Profile.Monitors.Any(monitor => monitor.Type == "IpmiUtilSensorMonitor")); - Assert.IsTrue(executor.Profile.Monitors.Any(monitor => monitor.Type == "IpmiUtilSelMonitor")); - } - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs deleted file mode 100644 index 9b9a3e620d..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/WindowsETWMonitorProfileTests.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Linq; - using NUnit.Framework; - using VirtualClient.Contracts; - using VirtualClient.TestExtensions; - - [TestFixture] - [Category("Functional")] - public class WindowsETWMonitorProfileTests - { - private DependencyFixture mockFixture; - - [OneTimeSetUp] - public void SetupFixture() - { - this.mockFixture = new DependencyFixture(); - ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); - } - - [Test] - [TestCase("MONITORS-ETW.json", 2)] - [TestCase("MONITORS-ETW-KERNEL.json", 1)] - [TestCase("MONITORS-ETW-NONKERNEL.json", 1)] - public void WindowsETWMonitorProfileParametersAreInlinedCorrectly(string profile, int expectedMonitorCount) - { - this.mockFixture.Setup(PlatformID.Win32NT); - - using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { - WorkloadAssert.ParameterReferencesInlined(executor.Profile); - Assert.AreEqual(expectedMonitorCount, executor.Profile.Monitors.Count); - Assert.IsTrue(executor.Profile.Monitors.All(monitor => monitor.Type == "WindowsETWMonitor")); - } - } - } -} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json deleted file mode 100644 index c9dae75011..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-AZURE-HOST.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "Description": "Default Azure Host Monitors", - "Metadata": { - "SupportedPlatforms": "win-x64,win-arm64", - "SupportedOperatingSystems": "Windows", - "PreliminaryRequirements": "This profile requires direct access to an IPMI-capable Azure host or blade." - }, - "Parameters": {}, - "Monitors": [ - { - "Type": "IpmiUtilSensorMonitor", - "Parameters": { - "Scenario": "CaptureSensorData", - "MonitorFrequency": "00:00:30" - } - }, - { - "Type": "IpmiUtilSelMonitor", - "Parameters": { - "Scenario": "CaptureSelData", - "ClearSel": false, - "EventsFilter": "Correctable ECC,FRB2,FRB3,IERR,Predictive failure,Uncorrectable ECC,Bus Error,Fatal NMI", - "MonitorFrequency": "00:01:00" - } - }, - { - "Type": "WindowsPerformanceCounterMonitor", - "Parameters": { - "Scenario": "CaptureCounters", - "MonitorFrequency": "00:05:00", - "MonitorWarmupPeriod": "00:05:00", - "Counters01": "Hyper-V Hypervisor Logical Processor=\\(_Total\\)", - "Counters02": "Hyper-V Hypervisor Root Virtual Processor=\\(_Total\\)\\\\% (Guest|Hypervisor|Total) Run Time", - "Counters03": "Hyper-V Hypervisor Root Virtual Processor=\\(_Total\\)\\\\(CPU Wait Time Per Dispatch|Hardware Interrupts/sec|Hypercalls Cost|Hypercalls/sec|Long Spin Wait Hypercalls/sec)", - "Counters04": "Hyper-V Hypervisor Root Virtual Processor=\\(_Total\\)\\\\(Total Intercepts Cost|Total Intercepts/sec)", - "Counters05": "Hyper-V Hypervisor Virtual Processor=\\(_Total\\)\\\\% (Guest|Hypervisor|Total|VTL2) Run Time", - "Counters06": "Hyper-V Hypervisor Virtual Processor=\\(_Total\\)\\\\(CPU Wait Time Per Dispatch|Hardware Interrupts/sec|Hypercalls Cost|Hypercalls/sec|Logical Processor Migrations/sec|Long Spin Wait Hypercalls/sec)", - "Counters07": "Hyper-V Hypervisor Virtual Processor=\\(_Total\\)\\\\(Nested Page Fault Intercepts/sec|Posted Interrupt Notifications/sec|Posted Interrupt Scans/sec|Total Intercepts Cost|Total Intercepts/sec)", - "Counters08": "IPv4=.", - "Counters09": "Memory=(Available|Cache|Committed) Bytes", - "Counters10": "Memory=Faults/sec", - "Counters11": "Memory=(Page Reads/sec|Page Writes/sec|Pages/sec|Pages Input/sec|Pages Output/sec)", - "Counters12": "PhysicalDisk=\\(_Total\\)", - "Counters13": "Processor Information=\\(_Total\\)", - "Counters14": "Processor Information=\\([0-9,]+\\)\\\\% (C[0-9]+|Idle|Interrupt|Privileged|Processor|User) Time", - "Counters15": "System=.", - "Comments": "The counters defined are formatted as {category}={counter_match_expression}. The match expression is a regular expression to allow for handling variability of counters available on different systems. JSON content has eccentricities. Use 4-backslashes to represent a single backslash and 2-backslashes to represent a regular expression character escape." - } - } - ] -} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json deleted file mode 100644 index 4f90aa962d..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-KERNEL.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "Description": "Capturing the ETW traces for Kernel provider", - "Metadata": { - "SupportedPlatforms": "win-x64,win-arm64", - "SupportedOperatingSystems": "Windows" - }, - "Parameters": { - "ProfilingEnabled": true, - "ProfilingMode": "Interval", - "ProfilingPeriod": "00:05:00", - "ProfilingInterval": "00:00:30", - "ProfilingWarmUpPeriod": "00:00:10" - }, - "Monitors": [ - { - "Type": "WindowsETWMonitor", - "Parameters": { - "Scenario": "CaptureKernelETWTraces", - "ProfilingEnabled": "$.Parameters.ProfilingEnabled", - "ProfilingMode": "$.Parameters.ProfilingMode", - "ProfilingPeriod": "$.Parameters.ProfilingPeriod", - "ProfilingInterval": "$.Parameters.ProfilingInterval", - "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", - "ProviderType": "Kernel", - "Provider": "Windows Kernel", - "ProviderKeywords": "ImageLoad,Process", - "ProviderEvents": "ImageLoad,ProcessStart,ProcessStop", - "JsonFilters": "" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json deleted file mode 100644 index 003f7b4340..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW-NONKERNEL.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "Description": "Capturing the ETW traces for Nonkernel provider", - "Metadata": { - "SupportedPlatforms": "win-x64,win-arm64", - "SupportedOperatingSystems": "Windows" - }, - "Parameters": { - "ProfilingEnabled": true, - "ProfilingMode": "Interval", - "ProfilingPeriod": "00:05:00", - "ProfilingInterval": "00:00:30", - "ProfilingWarmUpPeriod": "00:00:10" - }, - "Monitors": [ - { - "Type": "WindowsETWMonitor", - "Parameters": { - "Scenario": "CaptureNonKernelETWTraces", - "ProfilingEnabled": "$.Parameters.ProfilingEnabled", - "ProfilingMode": "$.Parameters.ProfilingMode", - "ProfilingPeriod": "$.Parameters.ProfilingPeriod", - "ProfilingInterval": "$.Parameters.ProfilingInterval", - "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", - "ProviderType": "NonKernel", - "Provider": "Microsoft-Windows-DotNETRuntime", - "JsonFilters": "{'EventName':['GCSetGCHandle','GCTriggered'],'Level':['4','2']}" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json deleted file mode 100644 index 6df147d78f..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-ETW.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "Description": "Capturing the ETW traces for Kernel and NonKernel events", - "Metadata": { - "SupportedPlatforms": "win-x64,win-arm64", - "SupportedOperatingSystems": "Windows" - }, - "Parameters": { - "ProfilingEnabled": true, - "ProfilingMode": "Interval", - "ProfilingPeriod": "00:05:00", - "ProfilingInterval": "00:00:30", - "ProfilingWarmUpPeriod": "00:00:10" - }, - "Monitors": [ - { - "Type": "WindowsETWMonitor", - "Parameters": { - "Scenario": "CaptureNonKernelETWTraces", - "ProfilingEnabled": "$.Parameters.ProfilingEnabled", - "ProfilingMode": "$.Parameters.ProfilingMode", - "ProfilingPeriod": "$.Parameters.ProfilingPeriod", - "ProfilingInterval": "$.Parameters.ProfilingInterval", - "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", - "ProviderType": "NonKernel", - "Provider": "Microsoft-Windows-DotNETRuntime", - "JsonFilters": "{'EventName':['GCSetGCHandle','GCTriggered'],'Level':['4','2']}" - - } - }, - { - "Type": "WindowsETWMonitor", - "Parameters": { - "Scenario": "CaptureKernelETWTraces", - "ProfilingEnabled": "$.Parameters.ProfilingEnabled", - "ProfilingMode": "$.Parameters.ProfilingMode", - "ProfilingPeriod": "$.Parameters.ProfilingPeriod", - "ProfilingInterval": "$.Parameters.ProfilingInterval", - "ProfilingWarmUpPeriod": "$.Parameters.ProfilingWarmUpPeriod", - "ProviderType": "Kernel", - "Provider": "Windows Kernel", - "ProviderKeywords": "ImageLoad,Process", - "ProviderEvents": "ImageLoad,ProcessStart,ProcessStop", - "JsonFilters": "" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json b/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json deleted file mode 100644 index bd4fc9ed57..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/MONITORS-IPMIUTIL.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "Description": "Captures BMC sensor readings and System Event Log (SEL) events using ipmiutil.", - "Metadata": { - "RecommendedMinimumExecutionTime": "00:05:00", - "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", - "SupportedOperatingSystems": "AzureLinux,CentOS,Debian,OpenSuse,RedHat,Ubuntu,Windows", - "PreliminaryRequirements": "The target system must expose a supported IPMI/BMC interface. Linux package installation requires access to the configured operating-system package repositories. Windows package installation requires access to the Virtual Client packages store." - }, - "Parameters": { - "MonitorFrequency": "00:01:00", - "MonitorWarmupPeriod": "00:00:00", - "MonitorIterations": -1, - "ClearSel": false, - "EventsFilter": "MAJ|CRT", - "CaptureSuspectedEvents": false, - "IncludeWarningsInSuspectedEvents": true, - "EnableSelDecoding": false, - "DecoderType": "", - "DecoderArguments": "", - "DecoderPackageName": "", - "DecoderExecutableName": "" - }, - "Monitors": [ - { - "Type": "IpmiUtilSensorMonitor", - "Parameters": { - "Scenario": "CaptureBmcSensorData", - "PackageName": "host.monitors", - "MonitorFrequency": "$.Parameters.MonitorFrequency", - "MonitorWarmupPeriod": "$.Parameters.MonitorWarmupPeriod", - "MonitorIterations": "$.Parameters.MonitorIterations" - } - }, - { - "Type": "IpmiUtilSelMonitor", - "Parameters": { - "Scenario": "CaptureBmcSelData", - "PackageName": "host.monitors", - "MonitorFrequency": "$.Parameters.MonitorFrequency", - "MonitorWarmupPeriod": "$.Parameters.MonitorWarmupPeriod", - "MonitorIterations": "$.Parameters.MonitorIterations", - "ClearSel": "$.Parameters.ClearSel", - "EventsFilter": "$.Parameters.EventsFilter", - "CaptureSuspectedEvents": "$.Parameters.CaptureSuspectedEvents", - "IncludeWarningsInSuspectedEvents": "$.Parameters.IncludeWarningsInSuspectedEvents", - "EnableSelDecoding": "$.Parameters.EnableSelDecoding", - "DecoderType": "$.Parameters.DecoderType", - "DecoderArguments": "$.Parameters.DecoderArguments", - "DecoderPackageName": "$.Parameters.DecoderPackageName", - "DecoderExecutableName": "$.Parameters.DecoderExecutableName" - } - } - ], - "Dependencies": [ - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallIpmiutilWindowsPackage", - "BlobContainer": "packages", - "BlobName": "host.monitors.1.0.1.zip", - "PackageName": "host.monitors", - "Extract": true, - "SupportedPlatforms": "win-x64,win-arm64" - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallIpmiutilLinuxPackage", - "Packages": "ipmiutil", - "AllowUpgrades": true, - "SupportedPlatforms": "linux-x64,linux-arm64" - } - } - ] -} diff --git a/website/docs/monitors/ipmiutil.md b/website/docs/monitors/ipmiutil.md index ed51788d72..b1da691d52 100644 --- a/website/docs/monitors/ipmiutil.md +++ b/website/docs/monitors/ipmiutil.md @@ -22,15 +22,8 @@ The system must expose an IPMI/BMC interface that IPMIUtil can access. The monit ## Installation -The `MONITORS-IPMIUTIL.json` profile installs IPMIUtil from the operating-system package repository on Linux. On Windows, it downloads `host.monitors.1.0.1.zip` from the Virtual Client `packages` container, so a package store must be supplied when the package is not already cached. - -``` bash -# Linux -sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 --logger=csv --log-to-file - -# Windows -VirtualClient.exe --profile=MONITORS-IPMIUTIL.json --timeout=30 --packages="" --logger=csv --log-to-file -``` +Install `ipmiutil` from the operating-system package repository on Linux. On Windows, install `host.monitors.1.0.1.zip` from the Virtual Client +`packages` container, supplying a package store when the package is not already cached. The usage example below shows both dependency definitions. The monitors search for `ipmiutil` or `ipmiutil.exe` on `PATH`. The SEL monitor also checks the legacy `C:\BladeFX_latest\BladeFX\Tools\IpmiUtil\ipmiutil.exe` location as a final fallback. @@ -101,15 +94,75 @@ Optional SEL decoding requires a separately supplied decoder package. The IPMIUt ## Usage Examples -``` bash +The following sections can be added to a custom execution profile: + +```json +"Parameters": { + "MonitorFrequency": "00:01:00", + "MonitorWarmupPeriod": "00:00:00", + "MonitorIterations": -1, + "ClearSel": false, + "EventsFilter": "MAJ|CRT" +}, +"Monitors": [ + { + "Type": "IpmiUtilSensorMonitor", + "Parameters": { + "Scenario": "CaptureBmcSensorData", + "PackageName": "host.monitors", + "MonitorFrequency": "$.Parameters.MonitorFrequency", + "MonitorWarmupPeriod": "$.Parameters.MonitorWarmupPeriod", + "MonitorIterations": "$.Parameters.MonitorIterations" + } + }, + { + "Type": "IpmiUtilSelMonitor", + "Parameters": { + "Scenario": "CaptureBmcSelData", + "PackageName": "host.monitors", + "MonitorFrequency": "$.Parameters.MonitorFrequency", + "MonitorWarmupPeriod": "$.Parameters.MonitorWarmupPeriod", + "MonitorIterations": "$.Parameters.MonitorIterations", + "ClearSel": "$.Parameters.ClearSel", + "EventsFilter": "$.Parameters.EventsFilter" + } + } +], +"Dependencies": [ + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallIpmiutilWindowsPackage", + "BlobContainer": "packages", + "BlobName": "host.monitors.1.0.1.zip", + "PackageName": "host.monitors", + "Extract": true, + "SupportedPlatforms": "win-x64,win-arm64" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallIpmiutilLinuxPackage", + "Packages": "ipmiutil", + "AllowUpgrades": true, + "SupportedPlatforms": "linux-x64,linux-arm64" + } + } +] +``` + +After saving the profile (for example, as `MONITORS-CUSTOM-IPMIUTIL.json`), run it with the desired parameters: + +```bash # Capture sensors and major/critical SEL events for 30 minutes. -sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 +sudo ./VirtualClient --profile=MONITORS-CUSTOM-IPMIUTIL.json --timeout=30 # Capture all decoded SEL severities without clearing the SEL. -sudo ./VirtualClient --profile=MONITORS-IPMIUTIL.json --timeout=30 --parameters="EventsFilter=.*;;;ClearSel=false" +sudo ./VirtualClient --profile=MONITORS-CUSTOM-IPMIUTIL.json --timeout=30 --parameters="EventsFilter=.*;;;ClearSel=false" -# Capture only one iteration after a 10-second warmup. -VirtualClient.exe --profile=MONITORS-IPMIUTIL.json --timeout=5 --parameters="MonitorIterations=1;;;MonitorWarmupPeriod=00:00:10" --packages="" +# Capture one iteration after a 10-second warmup on Windows. +VirtualClient.exe --profile=MONITORS-CUSTOM-IPMIUTIL.json --timeout=5 --parameters="MonitorIterations=1;;;MonitorWarmupPeriod=00:00:10" --packages="" ``` ## Troubleshooting diff --git a/website/docs/monitors/monitor-profiles.md b/website/docs/monitors/monitor-profiles.md index a7fcd5efad..dd8f8b40fa 100644 --- a/website/docs/monitors/monitor-profiles.md +++ b/website/docs/monitors/monitor-profiles.md @@ -8,13 +8,6 @@ The following sections describe the various monitor profiles that are available define the background monitors that will run on the system. Monitors are often ran in conjunction with workloads (defined in workload profiles) in order to capture performance and reliability information from the system while workloads are running. -## MONITORS-AZURE-HOST.json -This compatibility profile captures IPMIUtil sensors, BMC SEL records, and Windows performance counters on an Azure host or blade. It preserves the monitor names, intervals, event filter, and counter definitions from the profile previously distributed by the internal Virtual Client extension. - -The profile supports `win-x64` and `win-arm64` and requires direct access to an IPMI-capable physical host. IPMIUtil must already be available on `PATH`. The SEL monitor can also use the legacy BladeFX tool location. - -See the [IPMIUtil monitor documentation](/docs/monitors/ipmiutil) for hardware requirements and emitted telemetry. - ## MONITORS-DEFAULT.json The default monitor profile for the Virtual Client. This profile captures performance counters on the system using one or more different specialized toolsets. This monitor profile will be used when no other monitor profiles are specified on the command line. @@ -147,115 +140,3 @@ any of the dependencies required by the driver. If the driver is not already ins # Run the monitoring facilities only. ./VirtualClient --profile=MONITORS-GPU-NVIDIA.json --logger=csv --log-to-file ``` - -## MONITORS-ETW.json -This profile runs both the kernel and non-kernel [Windows ETW monitor](/docs/monitors/windows-etw) configurations. - -* **Supported Platform-Architectures** - * win-x64 - * win-arm64 - -* **Dependencies** - * Virtual Client must run with administrator privileges. - -* **Scenarios** - * `CaptureKernelETWTraces` captures process start, process stop, and image-load events from the Windows kernel provider. - * `CaptureNonKernelETWTraces` captures selected garbage-collection events from the .NET runtime provider. - -* **Profile Parameters** - - | Parameter | Purpose | Default value | - |-----------|---------|---------------| - | ProfilingEnabled | Enables or disables ETW collection. | `true` | - | ProfilingMode | Collection mode (`Interval` or `OnDemand`). | `Interval` | - | ProfilingPeriod | Length of each interval collection period. | `00:05:00` | - | ProfilingInterval | Delay between interval collection periods. | `00:00:30` | - | ProfilingWarmUpPeriod | Delay before interval collection starts. | `00:00:10` | - -* **Usage Examples** - - ```powershell - # Run both ETW monitor configurations and write telemetry locally. - VirtualClient.exe --profile=MONITORS-ETW.json --timeout=00:30:00 --logger=file - - # Run both ETW monitor configurations with a workload profile. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW.json --timeout=00:30:00 --logger=file - - # Upload ETW telemetry to an Event Hub. Internet connectivity is required. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW.json --timeout=00:30:00 --event-hub="" - ``` - -## MONITORS-ETW-KERNEL.json -This profile runs the [Windows ETW monitor](/docs/monitors/windows-etw) for the Windows kernel provider. It captures `ImageLoad`, -`ProcessStart`, and `ProcessStop` events enabled by the `ImageLoad` and `Process` keywords. - -* **Supported Platform-Architectures** - * win-x64 - * win-arm64 - -* **Dependencies** - * Virtual Client must run with administrator privileges. - -* **Profile Parameters** - See [MONITORS-ETW.json](#monitors-etwjson). - -* **Usage Examples** - - ```powershell - # Run the kernel ETW monitor and write telemetry locally. - VirtualClient.exe --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --logger=file - - # Run the kernel ETW monitor with a workload profile. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --logger=file - - # Upload kernel ETW telemetry to an Event Hub. Internet connectivity is required. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-KERNEL.json --timeout=00:30:00 --event-hub="" - ``` - -## MONITORS-ETW-NONKERNEL.json -This profile runs the [Windows ETW monitor](/docs/monitors/windows-etw) for the `Microsoft-Windows-DotNETRuntime` provider. It captures -`GCSetGCHandle` and `GCTriggered` events at the configured levels. - -* **Supported Platform-Architectures** - * win-x64 - * win-arm64 - -* **Dependencies** - * Virtual Client must run with administrator privileges. - -* **Profile Parameters** - See [MONITORS-ETW.json](#monitors-etwjson). - -* **Usage Examples** - - ```powershell - # Run the non-kernel ETW monitor and write telemetry locally. - VirtualClient.exe --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --logger=file - - # Run the non-kernel ETW monitor with a workload profile. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --logger=file - - # Upload non-kernel ETW telemetry to an Event Hub. Internet connectivity is required. - VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-ETW-NONKERNEL.json --timeout=00:30:00 --event-hub="" - ``` - -## MONITORS-IPMIUTIL.json -The IPMIUtil monitor profile captures BMC sensor measurements and System Event Log records throughout a Virtual Client run. - -* **Supported Platform-Architectures** - * linux-x64 - * linux-arm64 - * win-x64 - * win-arm64 - -* **Dependencies** - * A supported IPMI/BMC interface and sufficient privileges to access it. - * Linux access to an operating-system package repository that provides `ipmiutil`. - * Windows access to the Virtual Client package store containing `host.monitors.1.0.1.zip`. - -* **Scenarios** - * Capture numeric BMC sensor readings. - * Capture and combine decoded and raw BMC SEL records. - * Optionally preserve suspected hardware events and clear the SEL. - -See the [IPMIUtil monitor documentation](/docs/monitors/ipmiutil) for metrics, parameters, setup requirements, and examples. diff --git a/website/docs/monitors/systemevents/windows-etw.md b/website/docs/monitors/systemevents/windows-etw.md index 0f0a02910f..033bc877e3 100644 --- a/website/docs/monitors/systemevents/windows-etw.md +++ b/website/docs/monitors/systemevents/windows-etw.md @@ -41,8 +41,8 @@ The event information includes standard ETW fields such as `ActivityID`, `Provid `Task`, `TaskName`, `OpCode`, `OpCodeName`, `Keywords`, `TimeStamp`, `ProcessID`, `ProcessName`, `ProcessorNumber`, `ThreadID`, and `EventName`. Provider-specific payload fields are added using their ETW payload names. -When `--log-to-file` is specified, the configured Virtual Client logger writes these events to the local logs directory. Other configured -telemetry loggers can consume the same system-event records. +When `--logger=file` is specified, Virtual Client writes these events to the local logs directory. Other configured telemetry loggers can +consume the same system-event records. ## Parameters The following parameters are available for the `WindowsETWMonitor` component. @@ -58,7 +58,7 @@ The following parameters are available for the `WindowsETWMonitor` component. | ProfilingMode | No | Collection mode. Supported values are `Interval` and `OnDemand`. | `None` | | ProfilingPeriod | Interval mode | Length of each collection period. | `00:05:00` | | ProfilingInterval | Interval mode | Delay between collection periods. | `00:00:30` | -| ProfilingWarmUpPeriod | No | Delay before interval collection begins. On-demand requests supply their own warm-up period. | `00:00:10` in the provided profiles | +| ProfilingWarmUpPeriod | No | Delay before interval collection begins. On-demand requests supply their own warm-up period. | | | Scenario | No | Name describing the purpose of the monitor in the profile. | | `JsonFilters` values must be strings, including values that represent numbers. For example: @@ -68,7 +68,8 @@ The following parameters are available for the `WindowsETWMonitor` component. ``` ## Usage Examples -The following example captures process and image-load kernel events on an interval: +Add one or both of the following monitor entries to the `Monitors` section of an execution profile. The following example captures process and +image-load kernel events on an interval: ```json "Monitors": [ @@ -112,8 +113,18 @@ The following example captures selected .NET runtime events: ] ``` -Virtual Client includes ready-to-use profiles for kernel, non-kernel, and combined collection. See -[Monitor Profiles](/docs/monitors/monitor-profiles#monitors-etwjson). +After saving the profile (for example, as `MONITORS-CUSTOM-ETW.json`), run it by itself or with a workload: + +```powershell +# Run ETW monitoring and write telemetry locally. +VirtualClient.exe --profile=MONITORS-CUSTOM-ETW.json --timeout=00:30:00 --logger=file + +# Run ETW monitoring with a workload profile. +VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-CUSTOM-ETW.json --timeout=00:30:00 --logger=file + +# Upload ETW telemetry to an Event Hub. Internet connectivity is required. +VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-CUSTOM-ETW.json --timeout=00:30:00 --event-hub="" +``` ## Troubleshooting * **Unauthorized error** - Start Virtual Client from an elevated command prompt or PowerShell session.