Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions shared/source/os_interface/linux/drm_neo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
#include "shared/source/utilities/cpu_info.h"
#include "shared/source/utilities/directory.h"
#include "shared/source/utilities/io_functions.h"
#include "shared/source/utilities/logger.h"

#include "hw_cmds.h"

Expand Down Expand Up @@ -314,8 +315,16 @@ bool Drm::checkResetStatus(OsContext &osContext) {
printFault(stdout);
UNRECOVERABLE_IF(true);
}
if (contextHealth.banReason == ContextBanReason::gpuHang) {
PRINT_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "ERROR: GPU HANG detected!\n");
if (contextHealth.banReason != ContextBanReason::none) {
if (contextHealth.banReason == ContextBanReason::pageOffline) {
CREATE_DEBUG_STRING(str, "ERROR: GPU memory page offlined, ctx_id: %u (%s) banned, device lost.\n",
contextHealth.contextId,
EngineHelpers::engineTypeToString(osContext.getEngineType()).c_str());
PRINT_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", str.get());
rootDeviceEnvironment.executionEnvironment.setErrorDescription(std::string(str.get()));
} else {
PRINT_STRING(debugManager.flags.PrintDebugMessages.get(), stderr, "%s", "ERROR: GPU HANG detected!\n");
}
osContextLinux->setHangDetected();
return true;
}
Expand Down
1 change: 1 addition & 0 deletions shared/source/os_interface/linux/ioctl_helper.h
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ struct ContextFault {
enum class ContextBanReason : uint32_t {
none = 0,
gpuHang,
pageOffline,
};

struct ContextHealth {
Expand Down
8 changes: 7 additions & 1 deletion shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1406,7 +1406,13 @@ int IoctlHelperXe::getContextHealth(ContextHealth &contextHealth) {
return ret;
}
contextHealth.banned = getProperty.value != 0;
contextHealth.banReason = contextHealth.banned ? ContextBanReason::gpuHang : ContextBanReason::none;
if (!contextHealth.banned) {
contextHealth.banReason = ContextBanReason::none;
} else if ((getProperty.value & DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE) != 0) {
contextHealth.banReason = ContextBanReason::pageOffline;
} else {
contextHealth.banReason = ContextBanReason::gpuHang;
}

return 0;
}
Expand Down
10 changes: 10 additions & 0 deletions shared/test/common/mocks/linux/mock_ioctl_helper.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,15 @@ class MockIoctlHelper : public IoctlHelperPrelim20 {
return vmAdviseAtomicAttribute;
}

int getContextHealth(ContextHealth &contextHealth) override {
if (!contextBanReasonToReturn.has_value()) {
return IoctlHelperPrelim20::getContextHealth(contextHealth);
}
contextHealth.banned = *contextBanReasonToReturn != ContextBanReason::none;
contextHealth.banReason = *contextBanReasonToReturn;
return 0;
}

std::unique_ptr<MemoryInfo> createMemoryInfo() override {

std::vector<MemoryRegion> regionInfo(3);
Expand Down Expand Up @@ -95,6 +104,7 @@ class MockIoctlHelper : public IoctlHelperPrelim20 {
bool callBaseVmAdviseAtomicAttribute = true;
bool is2MBSizeAlignmentRequiredResult = false;
std::optional<uint32_t> vmAdviseAtomicAttribute{};
std::optional<ContextBanReason> contextBanReasonToReturn{};
void *pciBarrierMmapReturnValue = nullptr;
bool pciBarrierMmapCalled = false;
void *pciBarrierMmap() override {
Expand Down
58 changes: 58 additions & 0 deletions shared/test/unit_test/os_interface/linux/drm_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1681,6 +1681,64 @@ class MockIoctlHelperContextHealth : public MockIoctlHelper {
ContextFault faultReturnValue{};
};

struct DrmBanReasonTest : public ::testing::Test {
void SetUp() override {
drm = std::make_unique<DrmMock>(*executionEnvironment.rootDeviceEnvironments[0]);
auto engineDescriptor = EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_BCS, EngineUsage::regular});
osContext = std::make_unique<MockOsContextLinux>(*drm, 0, 0u, engineDescriptor);
osContext->drmContextIds.push_back(8);
auto ioctlHelper = std::make_unique<MockIoctlHelper>(*drm);
ioctlHelperPtr = ioctlHelper.get();
drm->ioctlHelper = std::move(ioctlHelper);
executionEnvironment.clearErrorDescription();
}

std::string checkResetStatusAndCaptureStderr() {
StreamCapture capture;
capture.captureStderr();
EXPECT_TRUE(drm->checkResetStatus(*osContext));
EXPECT_TRUE(osContext->isHangDetected());
return capture.getCapturedStderr();
}

const char *getErrorDescription() {
const char *errorDescription = nullptr;
executionEnvironment.getErrorDescription(&errorDescription);
return errorDescription;
}

DebugManagerStateRestore restore;
MockExecutionEnvironment executionEnvironment{};
std::unique_ptr<DrmMock> drm;
std::unique_ptr<MockOsContextLinux> osContext;
MockIoctlHelper *ioctlHelperPtr = nullptr;
const char *expectedPageOfflineMessage = "ERROR: GPU memory page offlined, ctx_id: 8 (BCS) banned, device lost.\n";
};

TEST_F(DrmBanReasonTest, givenPageOfflineBanReasonWhenCheckResetStatusIsCalledThenDeviceLostMessageIsPrintedAndErrorDescriptionIsSet) {
debugManager.flags.PrintDebugMessages.set(true);
ioctlHelperPtr->contextBanReasonToReturn = ContextBanReason::pageOffline;

EXPECT_STREQ(expectedPageOfflineMessage, checkResetStatusAndCaptureStderr().c_str());
EXPECT_STREQ(expectedPageOfflineMessage, getErrorDescription());
}

TEST_F(DrmBanReasonTest, givenPageOfflineBanReasonAndPrintDebugMessagesDisabledWhenCheckResetStatusIsCalledThenNothingIsPrintedButErrorDescriptionIsSet) {
debugManager.flags.PrintDebugMessages.set(false);
ioctlHelperPtr->contextBanReasonToReturn = ContextBanReason::pageOffline;

EXPECT_TRUE(checkResetStatusAndCaptureStderr().empty());
EXPECT_STREQ(expectedPageOfflineMessage, getErrorDescription());
}

TEST_F(DrmBanReasonTest, givenGpuHangBanReasonWhenCheckResetStatusIsCalledThenGpuHangIsReportedAndErrorDescriptionIsNotSet) {
debugManager.flags.PrintDebugMessages.set(true);
ioctlHelperPtr->contextBanReasonToReturn = ContextBanReason::gpuHang;

EXPECT_STREQ("ERROR: GPU HANG detected!\n", checkResetStatusAndCaptureStderr().c_str());
EXPECT_STREQ("", getErrorDescription());
}

TEST(DrmTest, GivenResetStatsWithValidFaultAndContextNotBannedAndDebuggingEnabledWhenIsGpuHangIsCalledThenProcessNotTerminated) {
DebugManagerStateRestore restore;
debugManager.flags.DisableScratchPages.set(true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3034,11 +3034,21 @@ TEST_F(IoctlHelperXeTest, whenCallingGetContextHealthThenBanPropertyDecidesTheBa
EXPECT_FALSE(contextHealth.banned);
EXPECT_EQ(ContextBanReason::none, contextHealth.banReason);

drm->execQueueBanPropertyReturn = 1;
drm->execQueueBanPropertyReturn = DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG;
EXPECT_EQ(0, xeIoctlHelper->getContextHealth(contextHealth));
EXPECT_TRUE(contextHealth.banned);
EXPECT_EQ(ContextBanReason::gpuHang, contextHealth.banReason);

drm->execQueueBanPropertyReturn = DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE;
EXPECT_EQ(0, xeIoctlHelper->getContextHealth(contextHealth));
EXPECT_TRUE(contextHealth.banned);
EXPECT_EQ(ContextBanReason::pageOffline, contextHealth.banReason);

drm->execQueueBanPropertyReturn = DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG | DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE;
EXPECT_EQ(0, xeIoctlHelper->getContextHealth(contextHealth));
EXPECT_TRUE(contextHealth.banned);
EXPECT_EQ(ContextBanReason::pageOffline, contextHealth.banReason);

// xe exposes no fault details through this property
EXPECT_FALSE(contextHealth.faultValid);
}
Expand Down
4 changes: 2 additions & 2 deletions third_party/uapi/drm-next/.version
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
git_revision: f5c6fdfa1f4460c5714cd4996c6aa280d25399ca
git_url: https://gitlab.freedesktop.org/drm/kernel.git
git_revision: c44e278ce02efd0c4be79a8eda1ea6885c1ce5ec
git_url: https://patchwork.freedesktop.org/patch/730877/?series=161473

12 changes: 11 additions & 1 deletion third_party/uapi/drm-next/xe/xe_drm.h
Original file line number Diff line number Diff line change
Expand Up @@ -1503,7 +1503,17 @@ struct drm_xe_exec_queue_get_property {
/** @property: property to get */
__u32 property;

/** @value: property value */
/**
* @value: property value
*
* For %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN, this is a bitmask of:
* - %DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG - banned due to GPU hang/timeout
* - %DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE - banned due to memory page offline
*
* Value of 0 means the exec queue is not banned.
*/
#define DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG (1 << 0)
#define DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE (1 << 1)
__u64 value;

/** @reserved: Reserved */
Expand Down