From 24b9fd4e67fed7e59f6dd581d31ab2fa554586eb Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:13:59 -0700 Subject: [PATCH 1/8] test: add real Falco container plugin regression corpus --- collector/CMakeLists.txt | 8 + collector/test/plugin-replay/CMakeLists.txt | 22 + collector/test/plugin-replay/CORPUS.md | 99 +++++ .../ContainerPluginReplayTest.cpp | 376 ++++++++++++++++++ collector/test/plugin-replay/Corpus.h | 101 +++++ collector/test/plugin-replay/README.md | 91 +++++ .../cgroup-order-candidate.patch | 12 + collector/test/plugin-replay/run-corpus.sh | 17 + 8 files changed, 726 insertions(+) create mode 100644 collector/test/plugin-replay/CMakeLists.txt create mode 100644 collector/test/plugin-replay/CORPUS.md create mode 100644 collector/test/plugin-replay/ContainerPluginReplayTest.cpp create mode 100644 collector/test/plugin-replay/Corpus.h create mode 100644 collector/test/plugin-replay/README.md create mode 100644 collector/test/plugin-replay/cgroup-order-candidate.patch create mode 100644 collector/test/plugin-replay/run-corpus.sh diff --git a/collector/CMakeLists.txt b/collector/CMakeLists.txt index adc11c1735..57e2b81602 100644 --- a/collector/CMakeLists.txt +++ b/collector/CMakeLists.txt @@ -104,4 +104,12 @@ set(MODERN_BPF_DEBUG_MODE ${BPF_DEBUG_MODE} CACHE BOOL "Enable BPF debug prints" set(MODERN_BPF_EXCLUDE_PROGS "^(openat2|ppoll|setsockopt|io_uring_setup|nanosleep|pread64|preadv|pwritev|read|readv|writev|recv|process_vm_readv|process_vm_writev)$" CACHE STRING "Set of syscalls to exclude from modern bpf engine " FORCE) +option(BUILD_PLUGIN_REPLAY_TESTS "Build isolated plugin replay tests" OFF) +if(BUILD_PLUGIN_REPLAY_TESTS) + set(HAS_ENGINE_TEST_INPUT ON) +endif() add_subdirectory(${FALCO_DIR} falco) + +if(BUILD_PLUGIN_REPLAY_TESTS) + add_subdirectory(test/plugin-replay) +endif() diff --git a/collector/test/plugin-replay/CMakeLists.txt b/collector/test/plugin-replay/CMakeLists.txt new file mode 100644 index 0000000000..0d7918f309 --- /dev/null +++ b/collector/test/plugin-replay/CMakeLists.txt @@ -0,0 +1,22 @@ +# Reuse the pinned upstream fixture without enabling its entire test suite. +find_package(GTest CONFIG REQUIRED) +set(FALCO_TEST_DIR "${FALCO_DIR}/userspace/libsinsp/test") +configure_file("${FALCO_TEST_DIR}/libsinsp_test_var.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/libsinsp_test_var.h") +file(GLOB FALCO_TEST_HELPERS "${FALCO_TEST_DIR}/helpers/*.cpp") +add_library(collector_replay_support STATIC + "${FALCO_TEST_DIR}/sinsp_with_test_input.cpp" + "${FALCO_TEST_DIR}/test_utils.cpp" + ${FALCO_TEST_HELPERS}) +target_include_directories(collector_replay_support PUBLIC + "${FALCO_DIR}" "${FALCO_TEST_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +target_link_libraries(collector_replay_support PUBLIC sinsp GTest::gtest) + +add_executable(ContainerPluginReplayTest ContainerPluginReplayTest.cpp) +target_link_libraries(ContainerPluginReplayTest PRIVATE + collector_replay_support collector_lib GTest::gtest_main) +add_dependencies(ContainerPluginReplayTest collector-container-plugin) +add_test(NAME ContainerPluginReplayTest COMMAND ContainerPluginReplayTest) +set_tests_properties(ContainerPluginReplayTest PROPERTIES + TIMEOUT 30 + ENVIRONMENT "ROX_COLLECTOR_CONTAINER_PLUGIN_PATH=$") diff --git a/collector/test/plugin-replay/CORPUS.md b/collector/test/plugin-replay/CORPUS.md new file mode 100644 index 0000000000..f856e306d8 --- /dev/null +++ b/collector/test/plugin-replay/CORPUS.md @@ -0,0 +1,99 @@ +# Plugin correctness corpus + +This is the first target of a Collector stability validator: real Falco, +production plugin callbacks, Collector attribution, and selected downstream +network handling. It is deterministic scenario replay, not random fuzzing yet. + +## Encoding and coverage + +`Corpus.h` contains typed C++ scenario data, with explicit expected IDs rather +than an oracle computed using the plugin's own extraction function. GTest +parameterization gives every generated case a stable readable name. + +- 16 startup cases: supported Docker/CRI-O/containerd/Podman layouts, host and + conmon exclusion, malformed IDs, and controller ordering. +- 33 fork cases: fork/clone/clone3; host, hostPID container, PID-namespace + container; parent-first, child-first, missing parent, and missing child events. +- 18 focused cases: original smoke tests; late imports; actual installed filter; + child-event recovery; exec/execveat and failed exec; TID reuse across containers + and from container to host; thread clone; valid vfork exit ordering; and two + production NetworkSignalHandler/ConnectionTracker tests. + +Every case constructs a fresh inspector and loads the plugin. Falco's real +TEST_INPUT engine performs parsing, thread creation, and plugin callbacks. No +test writes the cached container-ID field. Callback order is not mocked. + +The generated parent-only cases deliberately exclude PID-namespace parents: +their return value is a namespace-local TID, and Falco explicitly cannot create +the global child entry from that event alone. Parent-only host/hostPID cases +assert that Falco has a valid child and the expected cgroup before testing the +plugin's attribution. Their XML includes the cached ID and actual child cgroup. + +The vfork test observes the child, its exit, then the parent's return. Arbitrary +invalid vfork orderings are not treated as production defects. + +Late-import tests explicitly insert a valid thread through the real thread +manager after capture starts. This models the state resulting from a successful +late /proc lookup; TEST_INPUT does not implement live proc_get. Treat these two +cases as supporting evidence, not an end-to-end reproduction of live discovery. + +## Run + +Build using the commands in README.md. Then, inside the Linux builder: + +```sh +bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/corpus-results +``` + +The runner writes `run.log` and GTest/JUnit-style `results.xml`, and propagates +test failures as a nonzero exit status. It does not mark known defects as passing +or skip them. XML from a repeated GTest run describes its final iteration; the +log retains all iterations. + +Select a minimal reproduction with an ordinary GTest filter: + +```sh +bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/repro \ + --gtest_filter=ContainerPluginReplayTest.HostPIDConnectionTrackedWithoutChildForkEvent +``` + +For repeated sanitizer validation, build `/build-asan` using README.md, then: + +```sh +REPLAY_REPEAT=100 REPLAY_SEED=3939 \ +ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \ +UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ + bash /src/collector/test/plugin-replay/run-corpus.sh /build-asan /tmp/corpus-asan +``` + +An alternative plugin can be selected with +`ROX_COLLECTOR_CONTAINER_PLUGIN_PATH`. Use a compatible module from the same +Falco API build; this does not switch the linked Falco or Collector revision. + +## Observed results on original PR head 249fe750e + +67 cases: 54 pass, 13 fail. The failures represent two root causes, not thirteen +independent defects: + +1. Three cases reproduce a later cgroup overwriting an already found ID. +2. Eight cases reproduce missing cache initialization for a child created by + Falco's parent-side parser: six parameterized cases, installed host filtering, + and downstream connection tracking. Two additional late-import cases show + the same empty-cache behavior through explicitly modeled state insertion. + +The cgroup-only diagnostic patch fixes exactly three failures (57 pass, 10 fail), +confirming that cache initialization is independent of cgroup iteration order. + +The original plugin was run for 100 shuffled sanitizer iterations (6,700 test +executions); every iteration had the same 54/13 split, with no ASan/UBSan/leak +diagnostics. These are reproducible behavioral failures, not sanitizer crashes. + +TID reuse, exec/execveat refresh, failed exec, both observed fork orders, a missing +parent event, thread clone, valid vfork ordering, and the supported layout controls +passed. This does not establish exhaustive correctness or CPU performance. + +The downstream positive control records one connection with the correct ID when +both fork events are present. Omitting only the child event makes the production +handler return IGNORED and leaves the tracker empty. No Sensor service or network +delivery is involved; host filter bypass is not proof of a host signal reaching +Sensor. diff --git a/collector/test/plugin-replay/ContainerPluginReplayTest.cpp b/collector/test/plugin-replay/ContainerPluginReplayTest.cpp new file mode 100644 index 0000000000..e9d7c159ff --- /dev/null +++ b/collector/test/plugin-replay/ContainerPluginReplayTest.cpp @@ -0,0 +1,376 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +// The upstream fixture macro collides with protobuf's ErrorLocation enum. +#undef DEFAULT_VALUE + +#include "Corpus.h" +#include "NetworkSignalHandler.h" +#include "Utility.h" + +namespace collector { +namespace { + +const std::string kID(64, 'a'); +const std::string kPath = "/kubepods/burstable/pod123/" + kID; +const std::string kShortID = kID.substr(0, 12); + +class ContainerPluginReplayTest : public sinsp_with_test_input { + protected: + void SetUp() override { + const char* path = std::getenv("ROX_COLLECTOR_CONTAINER_PLUGIN_PATH"); + ASSERT_NE(path, nullptr); + plugin_ = m_inspector.register_plugin(path); + std::string error; + ASSERT_TRUE(plugin_->init("{}", error)) << error; + } + + void SeedThread(int64_t tid, int64_t vpid, + const std::vector& cgroups) { + auto thread = create_threadinfo(tid, tid, 0, tid, vpid, vpid, + "replay", "/bin/replay", "/bin/replay", + increasing_ts(), 0, 0, {}, 0, {}, "/"); + const auto bytes = test_utils::to_null_delimited(cgroups); + ASSERT_LE(bytes.size(), sizeof(thread.cgroups.path)); + std::memcpy(thread.cgroups.path, bytes.data(), bytes.size()); + thread.cgroups.len = bytes.size(); + add_thread(thread, {}); + } + + void Open() { + open_inspector(); // Falco invokes plugin_capture_open; tests never do so directly. + sinsp_filter_check_list checks; + checks.add_filter_check(m_inspector.new_generic_filtercheck()); + checks.add_filter_check(sinsp_plugin::new_filtercheck(plugin_)); + auto factory = std::make_shared(&m_inspector, checks); + sinsp_filter_compiler compiler(factory, "container.id != host"); + filter_ = compiler.compile(); + } + + void ExpectAttribution(int64_t tid, const std::string& expected, bool accepted) { + auto* event = generate_random_event(tid); + ASSERT_NE(event, nullptr); + EXPECT_EQ(GetContainerID(event), expected); + EXPECT_EQ(filter_->run(event), accepted); + } + + void ExpectConnection(int64_t tid, const std::string& expected, bool accepted) { + ASSERT_NE(generate_socket_exit_event({}, tid), nullptr); + auto* event = generate_connect_exit_event({}, tid); + ASSERT_NE(event, nullptr); + ASSERT_NE(event->get_thread_info(), nullptr); + ASSERT_FALSE(event->get_thread_info()->is_invalid()); + ASSERT_NE(event->get_fd_info(), nullptr); + EXPECT_EQ(GetContainerID(event), expected); + EXPECT_EQ(filter_->run(event), accepted); + } + + void ImportLateThread(int64_t tid, const std::vector& cgroups) { + // Models a successful late /proc lookup by importing its resulting state. + // TEST_INPUT has no proc_get callback; this is not an actual /proc lookup. + auto thread = m_inspector.get_threadinfo_factory().create(); + thread->m_tid = tid; + thread->m_pid = tid; + thread->m_ptid = 0; + thread->m_vtid = tid; + thread->m_vpid = tid; + thread->m_clone_ts = increasing_ts(); + thread->m_comm = "late-import"; + thread->m_exepath = "/bin/late-import"; + thread->set_cgroups(cgroups); + ASSERT_NE(m_inspector.m_thread_manager->add_thread(std::move(thread), false), nullptr); + } + + void ExpectHostPIDChildConnectionTracked(bool include_child_event) { + SeedThread(200, 200, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_clone_x_event(201, 200, 200, 0, 0, 200, 200, + "parent", {"memory=" + kPath}, PPME_SYSCALL_FORK_20_X), + nullptr); + if (include_child_event) { + ASSERT_NE(generate_clone_x_event(0, 201, 201, 200, 0, 201, 201, + "child", {"memory=" + kPath}, PPME_SYSCALL_FORK_20_X), + nullptr); + } + auto tracker = std::make_shared(); + system_inspector::Stats stats; + NetworkSignalHandler handler(&m_inspector, tracker, &stats); + ASSERT_NE(generate_socket_exit_event({}, 201), nullptr); + auto* event = generate_connect_exit_event({}, 201); + ASSERT_NE(event, nullptr); + ASSERT_TRUE(filter_->run(event)); + EXPECT_EQ(handler.HandleSignal(event), SignalHandler::PROCESSED); + const auto connections = tracker->FetchConnState(false, false); + ASSERT_EQ(connections.size(), 1U); + EXPECT_EQ(connections.begin()->first.container(), kShortID); + } + + std::shared_ptr plugin_; + std::unique_ptr filter_; +}; + +TEST_F(ContainerPluginReplayTest, StartupHostContainerAndHostPID) { + SeedThread(100, 100, {"memory=/"}); + SeedThread(200, 1, {"memory=" + kPath}); + SeedThread(300, 300, {"memory=" + kPath}); + Open(); + ExpectAttribution(100, "", false); + ExpectAttribution(200, kShortID, true); + ExpectAttribution(300, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, MatchingCgroupBeforeNonmatch) { + SeedThread(200, 1, {"memory=" + kPath, "cpuset=/"}); + Open(); + ExpectAttribution(200, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, MatchingCgroupAfterNonmatch) { + SeedThread(200, 1, {"cpuset=/", "memory=" + kPath}); + Open(); + ExpectAttribution(200, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, InstalledFilterRejectsHostAndAcceptsContainer) { + SeedThread(100, 100, {"memory=/"}); + SeedThread(200, 1, {"memory=" + kPath}); + Open(); + m_inspector.set_filter(std::move(filter_), "container.id != host"); + add_filtered_event_advance_ts(increasing_ts(), 100, PPME_SOCKET_GETSOCKNAME_X, 0); + auto* event = generate_random_event(200); + ASSERT_NE(event, nullptr); + EXPECT_EQ(GetContainerID(event), kShortID); +} + +TEST_F(ContainerPluginReplayTest, ForkWithoutExecAttributesFirstConnection) { + SeedThread(200, 1, {"memory=" + kPath}); + Open(); + // Replay parent and child fork exits; no exec event is generated. + ASSERT_NE(generate_clone_x_event(201, 200, 200, 0, 0, 1, 1, + "parent", {"memory=" + kPath}, + PPME_SYSCALL_FORK_20_X), + nullptr); + ASSERT_NE(generate_clone_x_event(0, 201, 201, 200, 0, 2, 2, + "child", {"memory=" + kPath}, + PPME_SYSCALL_FORK_20_X), + nullptr); + ASSERT_NE(generate_socket_exit_event({}, 201), nullptr); + auto* event = generate_connect_exit_event({}, 201); + ASSERT_NE(event, nullptr); + ASSERT_NE(event->get_fd_info(), nullptr); + EXPECT_EQ(GetContainerID(event), kShortID); + EXPECT_TRUE(filter_->run(event)); +} + +class StartupCorpusTest : public ContainerPluginReplayTest, + public ::testing::WithParamInterface {}; + +TEST_P(StartupCorpusTest, AttributionAndFilter) { + const auto& scenario = GetParam(); + SeedThread(200, 200, scenario.cgroups); + Open(); + ExpectAttribution(200, scenario.expected_id, !scenario.expected_id.empty()); +} + +INSTANTIATE_TEST_SUITE_P(Corpus, StartupCorpusTest, + ::testing::ValuesIn(replay_corpus::StartupCases()), + [](const auto& info) { return info.param.name; }); + +class ForkCorpusTest : public ContainerPluginReplayTest, + public ::testing::WithParamInterface {}; + +TEST_P(ForkCorpusTest, FirstConnectionAttributionAndFilter) { + using namespace replay_corpus; + const auto& scenario = GetParam(); + const bool container = scenario.origin != Origin::Host; + const bool pidns = scenario.origin == Origin::PIDNamespaceContainer; + const std::vector cgroups = {container ? "memory=" + kPath : "memory=/"}; + SeedThread(200, pidns ? 1 : 200, cgroups); + Open(); + const uint32_t flags = pidns ? PPM_CL_CHILD_IN_PIDNS : 0; + auto parent = [&]() { + ASSERT_NE(generate_clone_x_event(pidns ? 2 : 201, 200, 200, 0, flags, + pidns ? 1 : 200, pidns ? 1 : 200, + "parent", cgroups, scenario.event_type), + nullptr); + }; + auto child = [&]() { + ASSERT_NE(generate_clone_x_event(0, 201, 201, 200, flags, + pidns ? 2 : 201, pidns ? 2 : 201, + "child", cgroups, scenario.event_type), + nullptr); + }; + switch (scenario.order) { + case ForkOrder::ParentThenChild: + parent(); + child(); + break; + case ForkOrder::ChildThenParent: + child(); + parent(); + break; + case ForkOrder::ChildOnly: + child(); + break; + case ForkOrder::ParentOnly: + parent(); + break; + } + // Prove Falco already has enough information; missing attribution here cannot + // be excused by a missing thread or missing cgroup payload in the scenario. + auto thread = m_inspector.m_thread_manager->find_thread(201, true); + ASSERT_NE(thread, nullptr); + ASSERT_FALSE(thread->is_invalid()); + ASSERT_FALSE(thread->cgroups().empty()); + EXPECT_EQ(thread->cgroups().front().second, container ? kPath : "/"); + const auto& field = m_inspector.m_thread_manager->dynamic_fields()->fields().at("container_id"); + std::string cached_id; + thread->get_dynamic_field(field.new_accessor(), cached_id); + RecordProperty("cached_id_before_connection", cached_id); + RecordProperty("child_cgroup", thread->cgroups().front().second); + ExpectConnection(201, container ? kShortID : "", container); +} + +INSTANTIATE_TEST_SUITE_P(Corpus, ForkCorpusTest, + ::testing::ValuesIn(replay_corpus::ForkCases()), + [](const auto& info) { return info.param.name; }); + +TEST_F(ContainerPluginReplayTest, LateImportedHostIsRejected) { + SeedThread(1, 1, {"memory=/"}); + Open(); + ImportLateThread(200, {"memory=/"}); + ExpectConnection(200, "", false); +} + +TEST_F(ContainerPluginReplayTest, LateImportedContainerIsAttributed) { + SeedThread(1, 1, {"memory=/"}); + Open(); + ImportLateThread(200, {"memory=" + kPath}); + ExpectConnection(200, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, ParentForkHostChildRejectedByInstalledFilter) { + SeedThread(200, 200, {"memory=/"}); + Open(); + ASSERT_NE(generate_clone_x_event(201, 200, 200, 0, 0, 200, 200, + "parent", {"memory=/"}, PPME_SYSCALL_FORK_20_X), + nullptr); + auto child = m_inspector.m_thread_manager->find_thread(201, true); + ASSERT_NE(child, nullptr); + ASSERT_FALSE(child->is_invalid()); + m_inspector.set_filter(std::move(filter_), "container.id != host"); + // The child-side fork event was dropped; this is a socket event on the host. + add_filtered_event_advance_ts(increasing_ts(), 201, PPME_SOCKET_GETSOCKNAME_X, 0); +} + +TEST_F(ContainerPluginReplayTest, ChildForkRepairsPreviouslyUncachedHostPIDChild) { + SeedThread(200, 200, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_clone_x_event(201, 200, 200, 0, 0, 200, 200, + "parent", {"memory=" + kPath}, PPME_SYSCALL_FORK_20_X), + nullptr); + // Child traffic may precede the observed child-side event. Do not assert it + // here: the parent-only corpus separately checks that failing interval. + ASSERT_NE(generate_socket_exit_event({}, 201), nullptr); + ASSERT_NE(generate_clone_x_event(0, 201, 201, 200, 0, 201, 201, + "child", {"memory=" + kPath}, PPME_SYSCALL_FORK_20_X), + nullptr); + ExpectConnection(201, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, ExecRefreshesContainerAttribution) { + SeedThread(200, 1, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_execve_enter_and_exit_event(0, 200, 200, 200, 0, + "/bin/new", "new", "/bin/new", {"memory=/docker/" + replay_corpus::kB}), + nullptr); + ExpectConnection(200, "bbbbbbbbbbbb", true); +} + +TEST_F(ContainerPluginReplayTest, ExecveatRefreshesContainerAttribution) { + SeedThread(200, 1, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_execveat_enter_and_exit_event(0, 200, 200, 200, 0, + "/bin/new", "new", "/bin/new", {"memory=/docker/" + replay_corpus::kB}), + nullptr); + ExpectConnection(200, "bbbbbbbbbbbb", true); +} + +TEST_F(ContainerPluginReplayTest, FailedExecPreservesAttribution) { + SeedThread(200, 1, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_execve_enter_and_exit_event(-2, 200, 200, 200, 0, + "/missing", "missing", "/missing", {}), + nullptr); + ExpectConnection(200, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, TIDReuseAcrossContainersDoesNotRetainOldID) { + SeedThread(1, 1, {"memory=/"}); + SeedThread(200, 200, {"memory=" + kPath}); + Open(); + ExpectConnection(200, kShortID, true); + remove_thread(200, 1); + ASSERT_EQ(m_inspector.m_thread_manager->find_thread(200, true), nullptr); + ASSERT_NE(generate_clone_x_event(0, 200, 200, 1, 0, 200, 200, + "replacement", {"memory=/docker/" + replay_corpus::kB}, PPME_SYSCALL_FORK_20_X), + nullptr); + ExpectConnection(200, "bbbbbbbbbbbb", true); +} + +TEST_F(ContainerPluginReplayTest, TIDReuseFromContainerToHostIsRejected) { + SeedThread(1, 1, {"memory=/"}); + SeedThread(200, 200, {"memory=" + kPath}); + Open(); + remove_thread(200, 1); + ASSERT_EQ(m_inspector.m_thread_manager->find_thread(200, true), nullptr); + ASSERT_NE(generate_clone_x_event(0, 200, 200, 1, 0, 200, 200, + "replacement", {"memory=/"}, PPME_SYSCALL_FORK_20_X), + nullptr); + ExpectConnection(200, "", false); +} + +TEST_F(ContainerPluginReplayTest, ThreadCloneWithoutExecIsAttributed) { + SeedThread(200, 1, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_clone_x_event(0, 201, 200, 0, + PPM_CL_CLONE_THREAD | PPM_CL_CHILD_IN_PIDNS, 2, 1, + "worker", {"memory=" + kPath}, PPME_SYSCALL_CLONE_20_X), + nullptr); + ExpectConnection(201, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, VforkChildExitBeforeParentDoesNotResurrectChild) { + SeedThread(1, 1, {"memory=/"}); + SeedThread(200, 200, {"memory=" + kPath}); + Open(); + ASSERT_NE(generate_clone_x_event(0, 201, 201, 200, PPM_CL_CLONE_VFORK, 201, 201, + "child", {"memory=" + kPath}, PPME_SYSCALL_VFORK_20_X), + nullptr); + ExpectConnection(201, kShortID, true); + remove_thread(201, 200); + ASSERT_EQ(m_inspector.m_thread_manager->find_thread(201, true), nullptr); + ASSERT_NE(generate_clone_x_event(201, 200, 200, 0, PPM_CL_CLONE_VFORK, 200, 200, + "parent", {"memory=" + kPath}, PPME_SYSCALL_VFORK_20_X), + nullptr); + EXPECT_EQ(m_inspector.m_thread_manager->find_thread(201, true), nullptr); + ExpectConnection(200, kShortID, true); +} + +TEST_F(ContainerPluginReplayTest, HostPIDConnectionTrackedWithBothForkEvents) { + ExpectHostPIDChildConnectionTracked(true); +} + +TEST_F(ContainerPluginReplayTest, HostPIDConnectionTrackedWithoutChildForkEvent) { + ExpectHostPIDChildConnectionTracked(false); +} + +} // namespace +} // namespace collector diff --git a/collector/test/plugin-replay/Corpus.h b/collector/test/plugin-replay/Corpus.h new file mode 100644 index 0000000000..e1cf33a7f6 --- /dev/null +++ b/collector/test/plugin-replay/Corpus.h @@ -0,0 +1,101 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace collector::replay_corpus { + +// Explicit expectations: never derive the oracle with the plugin's extractor. +inline const std::string kA(64, 'a'); +inline const std::string kB(64, 'b'); +inline const std::string kContainer = "/kubepods/burstable/pod123/" + kA; + +struct StartupCase { + std::string name; + std::vector cgroups; + std::string expected_id; +}; + +inline void PrintTo(const StartupCase& scenario, std::ostream* out) { + *out << scenario.name; +} + +inline std::vector StartupCases() { + return { + {"Empty", {}, ""}, + {"HostRoot", {"memory=/", "cpuset=/"}, ""}, + {"HostSystemd", {"memory=/system.slice/kubelet.service"}, ""}, + {"DockerCgroupfs", {"memory=/docker/" + kA}, "aaaaaaaaaaaa"}, + {"DockerSystemd", {"memory=/system.slice/docker-" + kA + ".scope"}, "aaaaaaaaaaaa"}, + {"CrioSystemd", {"memory=/kubepods.slice/crio-" + kA + ".scope"}, "aaaaaaaaaaaa"}, + {"ContainerdSystemd", {"memory=/kubepods.slice/cri-containerd-" + kA + ".scope"}, "aaaaaaaaaaaa"}, + {"PodmanSystemd", {"memory=/machine.slice/libpod-" + kA + ".scope"}, "aaaaaaaaaaaa"}, + {"ConmonExcluded", {"memory=/machine.slice/libpod-conmon-" + kA + ".scope"}, ""}, + {"ShortIDRejected", {"memory=/docker/aaaaaaaaaaaa"}, ""}, + {"NonhexRejected", {"memory=/docker/" + std::string(64, 'z')}, ""}, + {"InvalidSeparatorRejected", {"memory=/docker_" + kA}, ""}, + {"SameIDAcrossControllers", {"memory=" + kContainer, "cpu=" + kContainer}, "aaaaaaaaaaaa"}, + {"MatchThenHost", {"memory=" + kContainer, "cpuset=/"}, "aaaaaaaaaaaa"}, + {"HostThenMatch", {"cpuset=/", "memory=" + kContainer}, "aaaaaaaaaaaa"}, + {"MatchBetweenHosts", {"cpu=/", "memory=" + kContainer, "cpuset=/"}, "aaaaaaaaaaaa"}, + }; +} + +enum class Origin { Host, + HostPIDContainer, + PIDNamespaceContainer }; +enum class ForkOrder { ParentThenChild, + ChildThenParent, + ChildOnly, + ParentOnly }; + +struct ForkCase { + std::string name; + ppm_event_code event_type; + Origin origin; + ForkOrder order; +}; + +inline void PrintTo(const ForkCase& scenario, std::ostream* out) { + *out << scenario.name; +} + +inline std::vector ForkCases() { + std::vector cases; + const std::vector> events = { + {"Fork", PPME_SYSCALL_FORK_20_X}, + {"Clone", PPME_SYSCALL_CLONE_20_X}, + {"Clone3", PPME_SYSCALL_CLONE3_X}, + }; + const std::vector> origins = { + {"Host", Origin::Host}, + {"HostPID", Origin::HostPIDContainer}, + {"PIDNamespace", Origin::PIDNamespaceContainer}, + }; + const std::vector> orders = { + {"ParentThenChild", ForkOrder::ParentThenChild}, + {"ChildThenParent", ForkOrder::ChildThenParent}, + {"ChildOnly", ForkOrder::ChildOnly}, + {"ParentOnly", ForkOrder::ParentOnly}, + }; + for (const auto& [event_name, event] : events) { + for (const auto& [origin_name, origin] : origins) { + for (const auto& [order_name, order] : orders) { + // A parent in a PID namespace cannot supply the global child TID. + // Falco deliberately waits for the child event; no valid-ID oracle here. + if (origin == Origin::PIDNamespaceContainer && order == ForkOrder::ParentOnly) { + continue; + } + cases.push_back({event_name + "_" + origin_name + "_" + order_name, + event, origin, order}); + } + } + } + return cases; +} + +} // namespace collector::replay_corpus diff --git a/collector/test/plugin-replay/README.md b/collector/test/plugin-replay/README.md new file mode 100644 index 0000000000..f937c51bcb --- /dev/null +++ b/collector/test/plugin-replay/README.md @@ -0,0 +1,91 @@ +# Plugin correctness validator + +An opt-in validator that links production `collector_lib`, loads the actual +container plugin, and replays synthetic events through the pinned Falco library. +No live capture, container runtime socket, or Kubernetes cluster is needed. +See [CORPUS.md](CORPUS.md) for the 67 scenarios and expected outcomes. + +## Build and run locally + +In a Linux Collector builder environment with source at `/src`: + +```sh +cmake -S /src -B /build \ + -DBUILD_PLUGIN_REPLAY_TESTS=ON \ + -DCMAKE_BUILD_TYPE=Debug -DDISABLE_PROFILING=ON +cmake --build /build --target ContainerPluginReplayTest -j4 +bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/plugin-results +``` + +Initialize the pinned `falcosecurity-libs` and +`collector/proto/third_party/stackrox` submodules first. Falco sources must be +writable because its build generates some source-tree headers. + +The runner retains logs and XML and returns nonzero on test failure. CTest also +registers `ContainerPluginReplayTest` and supplies the plugin path automatically: + +```sh +ctest --test-dir /build -R '^ContainerPluginReplayTest$' --output-on-failure +``` + +The build reuses upstream fixture sources without enabling the entire upstream +test suite. It enables Falco's TEST_INPUT engine only with this build option. +Normal Collector test builds remain unchanged when the option is off. + +## Sanitizers + +Instrument C as well as C++ so libscap participates: + +```sh +cmake -S /src -B /build-asan \ + -DBUILD_PLUGIN_REPLAY_TESTS=ON -DADDRESS_SANITIZER=ON \ + -DCMAKE_BUILD_TYPE=Debug -DDISABLE_PROFILING=ON \ + '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer' +cmake --build /build-asan --target ContainerPluginReplayTest -j2 +REPLAY_REPEAT=100 REPLAY_SEED=3939 \ +ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \ +UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ + bash /src/collector/test/plugin-replay/run-corpus.sh /build-asan /tmp/plugin-asan +``` + +External prebuilt builder libraries are not rebuilt with instrumentation. If a +Debug build already generated the BPF skeleton, the sanitizer build can reuse it +with `-DMODERN_BPF_SKEL_DIR=/build/skel_dir`. No BPF program is loaded by replay. + +## CI + +`.github/workflows/plugin-validator.yml` runs on pull requests targeting +`collector-container-plugin` when relevant sources change, including draft PRs. +It builds the exact PR head against its pinned submodules in a pinned multiarch +Collector builder image. AMD64 and ARM64 each run 100 shuffled ASan/UBSan +iterations. Logs, XML, and revision information are uploaded even if replay fails. +The workflow is independent of the main image/integration-test pipeline and uses +only read access to repository contents. + +The expected original-PR result is red: 54/67 cases pass and 13 fail. Known +regressions are not skipped or converted to success. Fixes belong in the parent +plugin branch; updating this stacked branch and rerunning provides validation. + +## Diagnostic candidate and limitations + +`cgroup-order-candidate.patch` preserves the first nonempty ID. It is retained +only as a diagnostic artifact and is never applied by the build or CI. It fixes +three failures (57/67 pass), leaving the cache lifecycle failures unresolved. +From the repository root, an isolated local check can use: + +```sh +git apply --check collector/test/plugin-replay/cgroup-order-candidate.patch +git apply collector/test/plugin-replay/cgroup-order-candidate.patch +cmake --build /build --target ContainerPluginReplayTest -j4 +bash collector/test/plugin-replay/run-corpus.sh /build /tmp/plugin-candidate +# Once the test completes (its failures are expected), restore the source: +git apply -R collector/test/plugin-replay/cgroup-order-candidate.patch +cmake --build /build --target ContainerPluginReplayTest -j4 +``` + +The tests use real Falco callbacks and production ID lookup; two also exercise +the network handler and connection tracker. The short filter-construction code +is currently duplicated from Service.cpp, not shared with service startup. +Late-import cases model the result of a successful /proc lookup because TEST_INPUT +has no live proc_get callback. There is no Sensor delivery, random mutation, or +performance benchmark here. These boundaries are detailed in CORPUS.md. diff --git a/collector/test/plugin-replay/cgroup-order-candidate.patch b/collector/test/plugin-replay/cgroup-order-candidate.patch new file mode 100644 index 0000000000..d041138842 --- /dev/null +++ b/collector/test/plugin-replay/cgroup-order-candidate.patch @@ -0,0 +1,12 @@ +diff --git a/collector/container-plugin/ContainerPlugin.cpp b/collector/container-plugin/ContainerPlugin.cpp +--- a/collector/container-plugin/ContainerPlugin.cpp ++++ b/collector/container-plugin/ContainerPlugin.cpp +@@ -41,7 +41,7 @@ + if (state->reader->read_entry_field(state->cgroup_table, entry, state->plugin->cgroup_path, &value) != SS_PLUGIN_SUCCESS) { + return 0; + } +- if (value.str != nullptr) { ++ if (value.str != nullptr && state->container_id.empty()) { + state->container_id = ContainerIDFromCgroup(value.str); + } + return 1; diff --git a/collector/test/plugin-replay/run-corpus.sh b/collector/test/plugin-replay/run-corpus.sh new file mode 100644 index 0000000000..d4f7a5a19a --- /dev/null +++ b/collector/test/plugin-replay/run-corpus.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run an already-built validator and retain its actual exit status and evidence. +set -euo pipefail + +build_dir="${1:?Usage: bash run-corpus.sh BUILD_DIR RESULTS_DIR [gtest arguments...]}" +results_dir="${2:?Usage: bash run-corpus.sh BUILD_DIR RESULTS_DIR [gtest arguments...]}" +shift 2 + +test -x "$build_dir/collector/test/plugin-replay/ContainerPluginReplayTest" +mkdir -p "$results_dir" +export ROX_COLLECTOR_CONTAINER_PLUGIN_PATH="${ROX_COLLECTOR_CONTAINER_PLUGIN_PATH:-$build_dir/collector/collector-container-plugin.so}" + +"$build_dir/collector/test/plugin-replay/ContainerPluginReplayTest" \ + --gtest_repeat="${REPLAY_REPEAT:-1}" \ + --gtest_shuffle --gtest_random_seed="${REPLAY_SEED:-3939}" \ + --gtest_output="xml:$results_dir/results.xml" \ + "$@" 2>&1 | tee "$results_dir/run.log" From 1967b1232d0732a1fdf534072dba4daa79967567 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:13:59 -0700 Subject: [PATCH 2/8] ci: run plugin validator on stacked pull requests --- .github/workflows/plugin-validator.yml | 89 ++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/plugin-validator.yml diff --git a/.github/workflows/plugin-validator.yml b/.github/workflows/plugin-validator.yml new file mode 100644 index 0000000000..2a7c31e2a9 --- /dev/null +++ b/.github/workflows/plugin-validator.yml @@ -0,0 +1,89 @@ +name: Plugin correctness validator + +on: + pull_request: + branches: [collector-container-plugin] + types: [opened, synchronize, reopened, ready_for_review] + paths: + - collector/** + - falcosecurity-libs + - .github/workflows/plugin-validator.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: plugin-validator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + replay: + name: Plugin corpus (ASan/UBSan, ${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + container: + # Multiarch digest of the builder used for local corpus validation. + image: quay.io/stackrox-io/collector-builder@sha256:4ae0b728c58b38b67cdb426704a1e29d118ee1f979ce71606687874b4f5fa272 + defaults: + run: + shell: bash + env: + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + REPLAY_REPEAT: '100' + REPLAY_SEED: '3939' + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Initialize required submodules and record revisions + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git submodule update --init --depth 1 falcosecurity-libs collector/proto/third_party/stackrox + mkdir -p artifacts + { + git rev-parse HEAD + git submodule status falcosecurity-libs collector/proto/third_party/stackrox + uname -sm + c++ --version + } | tee artifacts/revisions.txt + + - name: Configure sanitizer replay target + run: | + cmake -S . -B cmake-build \ + -DBUILD_PLUGIN_REPLAY_TESTS=ON \ + -DADDRESS_SANITIZER=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -DDISABLE_PROFILING=ON \ + '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer' \ + 2>&1 | tee artifacts/configure.log + + - name: Build validator and production plugin + run: | + cmake --build cmake-build --target ContainerPluginReplayTest --parallel 2 \ + 2>&1 | tee artifacts/build.log + + - name: Replay corpus with shuffled repetitions + run: | + bash collector/test/plugin-replay/run-corpus.sh \ + "$GITHUB_WORKSPACE/cmake-build" "$GITHUB_WORKSPACE/artifacts/replay" + + - name: Upload validation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: plugin-validator-${{ matrix.arch }}-${{ github.run_attempt }} + path: artifacts/ + if-no-files-found: warn + retention-days: 14 From 2a8d0c43904751336d4d1261f9fefa57cf14ef4e Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:17:40 -0700 Subject: [PATCH 3/8] ci: reuse main builder selection and keep validator test-only --- .github/workflows/main.yml | 7 ++++ .github/workflows/plugin-validator.yml | 32 ++++++------------- collector/test/plugin-replay/CORPUS.md | 3 -- collector/test/plugin-replay/README.md | 29 +++++------------ .../cgroup-order-candidate.patch | 12 ------- 5 files changed, 25 insertions(+), 58 deletions(-) delete mode 100644 collector/test/plugin-replay/cgroup-order-candidate.patch diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f002e6eb7b..7c131bcd20 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,6 +71,13 @@ jobs: collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} secrets: inherit + plugin-validator: + uses: ./.github/workflows/plugin-validator.yml + needs: + - build-builder-image + with: + collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + integration-tests: uses: ./.github/workflows/integration-tests.yml with: diff --git a/.github/workflows/plugin-validator.yml b/.github/workflows/plugin-validator.yml index 2a7c31e2a9..2a7d87c0ec 100644 --- a/.github/workflows/plugin-validator.yml +++ b/.github/workflows/plugin-validator.yml @@ -1,38 +1,27 @@ name: Plugin correctness validator on: - pull_request: - branches: [collector-container-plugin] - types: [opened, synchronize, reopened, ready_for_review] - paths: - - collector/** - - falcosecurity-libs - - .github/workflows/plugin-validator.yml - workflow_dispatch: + workflow_call: + inputs: + collector-builder-tag: + type: string + required: true + description: The builder tag to use in the build permissions: contents: read -concurrency: - group: plugin-validator-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: replay: name: Plugin corpus (ASan/UBSan, ${{ matrix.arch }}) strategy: fail-fast: false matrix: - include: - - arch: amd64 - runner: ubuntu-24.04 - - arch: arm64 - runner: ubuntu-24.04-arm - runs-on: ${{ matrix.runner }} + arch: [amd64, arm64] + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} timeout-minutes: 45 container: - # Multiarch digest of the builder used for local corpus validation. - image: quay.io/stackrox-io/collector-builder@sha256:4ae0b728c58b38b67cdb426704a1e29d118ee1f979ce71606687874b4f5fa272 + image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} defaults: run: shell: bash @@ -44,8 +33,6 @@ jobs: steps: - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Initialize required submodules and record revisions run: | @@ -55,6 +42,7 @@ jobs: { git rev-parse HEAD git submodule status falcosecurity-libs collector/proto/third_party/stackrox + echo "Builder: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }}" uname -sm c++ --version } | tee artifacts/revisions.txt diff --git a/collector/test/plugin-replay/CORPUS.md b/collector/test/plugin-replay/CORPUS.md index f856e306d8..0fbd6c16e2 100644 --- a/collector/test/plugin-replay/CORPUS.md +++ b/collector/test/plugin-replay/CORPUS.md @@ -81,9 +81,6 @@ independent defects: and downstream connection tracking. Two additional late-import cases show the same empty-cache behavior through explicitly modeled state insertion. -The cgroup-only diagnostic patch fixes exactly three failures (57 pass, 10 fail), -confirming that cache initialization is independent of cgroup iteration order. - The original plugin was run for 100 shuffled sanitizer iterations (6,700 test executions); every iteration had the same 54/13 split, with no ASan/UBSan/leak diagnostics. These are reproducible behavioral failures, not sanitizer crashes. diff --git a/collector/test/plugin-replay/README.md b/collector/test/plugin-replay/README.md index f937c51bcb..ac9a6ba231 100644 --- a/collector/test/plugin-replay/README.md +++ b/collector/test/plugin-replay/README.md @@ -54,34 +54,21 @@ with `-DMODERN_BPF_SKEL_DIR=/build/skel_dir`. No BPF program is loaded by replay ## CI -`.github/workflows/plugin-validator.yml` runs on pull requests targeting -`collector-container-plugin` when relevant sources change, including draft PRs. -It builds the exact PR head against its pinned submodules in a pinned multiarch -Collector builder image. AMD64 and ARM64 each run 100 shuffled ASan/UBSan +Main CI calls `.github/workflows/plugin-validator.yml` alongside unit tests, +using the same `build-builder-image` output tag and standard checkout (the merge +commit for pull requests). It builds against that revision's pinned submodules. +AMD64 and ARM64 each run 100 shuffled ASan/UBSan iterations. Logs, XML, and revision information are uploaded even if replay fails. -The workflow is independent of the main image/integration-test pipeline and uses -only read access to repository contents. +The validator needs only the builder job, not the Collector image or integration +tests, and uses only read access to repository contents. The expected original-PR result is red: 54/67 cases pass and 13 fail. Known regressions are not skipped or converted to success. Fixes belong in the parent plugin branch; updating this stacked branch and rerunning provides validation. -## Diagnostic candidate and limitations +## Limitations -`cgroup-order-candidate.patch` preserves the first nonempty ID. It is retained -only as a diagnostic artifact and is never applied by the build or CI. It fixes -three failures (57/67 pass), leaving the cache lifecycle failures unresolved. -From the repository root, an isolated local check can use: - -```sh -git apply --check collector/test/plugin-replay/cgroup-order-candidate.patch -git apply collector/test/plugin-replay/cgroup-order-candidate.patch -cmake --build /build --target ContainerPluginReplayTest -j4 -bash collector/test/plugin-replay/run-corpus.sh /build /tmp/plugin-candidate -# Once the test completes (its failures are expected), restore the source: -git apply -R collector/test/plugin-replay/cgroup-order-candidate.patch -cmake --build /build --target ContainerPluginReplayTest -j4 -``` +This contribution contains tests and CI only; plugin fixes belong in the parent PR. The tests use real Falco callbacks and production ID lookup; two also exercise the network handler and connection tracker. The short filter-construction code diff --git a/collector/test/plugin-replay/cgroup-order-candidate.patch b/collector/test/plugin-replay/cgroup-order-candidate.patch deleted file mode 100644 index d041138842..0000000000 --- a/collector/test/plugin-replay/cgroup-order-candidate.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/collector/container-plugin/ContainerPlugin.cpp b/collector/container-plugin/ContainerPlugin.cpp ---- a/collector/container-plugin/ContainerPlugin.cpp -+++ b/collector/container-plugin/ContainerPlugin.cpp -@@ -41,7 +41,7 @@ - if (state->reader->read_entry_field(state->cgroup_table, entry, state->plugin->cgroup_path, &value) != SS_PLUGIN_SUCCESS) { - return 0; - } -- if (value.str != nullptr) { -+ if (value.str != nullptr && state->container_id.empty()) { - state->container_id = ContainerIDFromCgroup(value.str); - } - return 1; From e0a25e9c24118ac2c7176d483315c026a8092550 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:18:12 -0700 Subject: [PATCH 4/8] ci: grant validator caller read-only contents access --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c131bcd20..0c96258726 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,6 +73,8 @@ jobs: plugin-validator: uses: ./.github/workflows/plugin-validator.yml + permissions: + contents: read needs: - build-builder-image with: From ec2f6ce7a9ce8b02fc21097d36404c0e332d9842 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:19:22 -0700 Subject: [PATCH 5/8] docs: consolidate validator approach corpus and tradeoffs --- collector/test/plugin-replay/CORPUS.md | 96 ------------------- collector/test/plugin-replay/README.md | 125 ++++++++++++++++++++++--- 2 files changed, 111 insertions(+), 110 deletions(-) delete mode 100644 collector/test/plugin-replay/CORPUS.md diff --git a/collector/test/plugin-replay/CORPUS.md b/collector/test/plugin-replay/CORPUS.md deleted file mode 100644 index 0fbd6c16e2..0000000000 --- a/collector/test/plugin-replay/CORPUS.md +++ /dev/null @@ -1,96 +0,0 @@ -# Plugin correctness corpus - -This is the first target of a Collector stability validator: real Falco, -production plugin callbacks, Collector attribution, and selected downstream -network handling. It is deterministic scenario replay, not random fuzzing yet. - -## Encoding and coverage - -`Corpus.h` contains typed C++ scenario data, with explicit expected IDs rather -than an oracle computed using the plugin's own extraction function. GTest -parameterization gives every generated case a stable readable name. - -- 16 startup cases: supported Docker/CRI-O/containerd/Podman layouts, host and - conmon exclusion, malformed IDs, and controller ordering. -- 33 fork cases: fork/clone/clone3; host, hostPID container, PID-namespace - container; parent-first, child-first, missing parent, and missing child events. -- 18 focused cases: original smoke tests; late imports; actual installed filter; - child-event recovery; exec/execveat and failed exec; TID reuse across containers - and from container to host; thread clone; valid vfork exit ordering; and two - production NetworkSignalHandler/ConnectionTracker tests. - -Every case constructs a fresh inspector and loads the plugin. Falco's real -TEST_INPUT engine performs parsing, thread creation, and plugin callbacks. No -test writes the cached container-ID field. Callback order is not mocked. - -The generated parent-only cases deliberately exclude PID-namespace parents: -their return value is a namespace-local TID, and Falco explicitly cannot create -the global child entry from that event alone. Parent-only host/hostPID cases -assert that Falco has a valid child and the expected cgroup before testing the -plugin's attribution. Their XML includes the cached ID and actual child cgroup. - -The vfork test observes the child, its exit, then the parent's return. Arbitrary -invalid vfork orderings are not treated as production defects. - -Late-import tests explicitly insert a valid thread through the real thread -manager after capture starts. This models the state resulting from a successful -late /proc lookup; TEST_INPUT does not implement live proc_get. Treat these two -cases as supporting evidence, not an end-to-end reproduction of live discovery. - -## Run - -Build using the commands in README.md. Then, inside the Linux builder: - -```sh -bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/corpus-results -``` - -The runner writes `run.log` and GTest/JUnit-style `results.xml`, and propagates -test failures as a nonzero exit status. It does not mark known defects as passing -or skip them. XML from a repeated GTest run describes its final iteration; the -log retains all iterations. - -Select a minimal reproduction with an ordinary GTest filter: - -```sh -bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/repro \ - --gtest_filter=ContainerPluginReplayTest.HostPIDConnectionTrackedWithoutChildForkEvent -``` - -For repeated sanitizer validation, build `/build-asan` using README.md, then: - -```sh -REPLAY_REPEAT=100 REPLAY_SEED=3939 \ -ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \ -UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ - bash /src/collector/test/plugin-replay/run-corpus.sh /build-asan /tmp/corpus-asan -``` - -An alternative plugin can be selected with -`ROX_COLLECTOR_CONTAINER_PLUGIN_PATH`. Use a compatible module from the same -Falco API build; this does not switch the linked Falco or Collector revision. - -## Observed results on original PR head 249fe750e - -67 cases: 54 pass, 13 fail. The failures represent two root causes, not thirteen -independent defects: - -1. Three cases reproduce a later cgroup overwriting an already found ID. -2. Eight cases reproduce missing cache initialization for a child created by - Falco's parent-side parser: six parameterized cases, installed host filtering, - and downstream connection tracking. Two additional late-import cases show - the same empty-cache behavior through explicitly modeled state insertion. - -The original plugin was run for 100 shuffled sanitizer iterations (6,700 test -executions); every iteration had the same 54/13 split, with no ASan/UBSan/leak -diagnostics. These are reproducible behavioral failures, not sanitizer crashes. - -TID reuse, exec/execveat refresh, failed exec, both observed fork orders, a missing -parent event, thread clone, valid vfork ordering, and the supported layout controls -passed. This does not establish exhaustive correctness or CPU performance. - -The downstream positive control records one connection with the correct ID when -both fork events are present. Omitting only the child event makes the production -handler return IGNORED and leaves the tracker empty. No Sensor service or network -delivery is involved; host filter bypass is not proof of a host signal reaching -Sensor. diff --git a/collector/test/plugin-replay/README.md b/collector/test/plugin-replay/README.md index ac9a6ba231..2d66476f5c 100644 --- a/collector/test/plugin-replay/README.md +++ b/collector/test/plugin-replay/README.md @@ -1,9 +1,77 @@ # Plugin correctness validator -An opt-in validator that links production `collector_lib`, loads the actual -container plugin, and replays synthetic events through the pinned Falco library. -No live capture, container runtime socket, or Kubernetes cluster is needed. -See [CORPUS.md](CORPUS.md) for the 67 scenarios and expected outcomes. +The container plugin caches which container a process belongs to. Collector uses +that answer to exclude host activity and attribute container connections. A wrong +or missing answer can silently drop a legitimate connection or let host activity +through the filter. This validator checks those decisions before we assess speed. + +It is a 67-case deterministic regression suite, not a random fuzzer. This PR adds +tests and CI only; plugin fixes belong in the parent PR. + +## How it works + +Each test gives a fresh Falco inspector a small process inventory and a sequence +of synthetic events, such as a process starting a child and that child connecting +to a socket. Falco's real TEST_INPUT engine parses those events and invokes the +actual compiled plugin. We check the resulting container ID, filtering decision, +and, in two tests, Collector's network handler and connection tracker. + +The path under test is: + +`scenario -> real Falco parser -> real plugin -> Collector attribution/filter -> selected network checks` + +We synthesize the input, not the plugin's answer. Tests never write the plugin's +cached container-ID field. Expected IDs are explicit scenario data, not values +computed with the plugin's own extraction logic. There is no live kernel capture, +container runtime socket, Kubernetes cluster, or Sensor service. + +Why simulate process IDs? A fork can be observed from both the parent and child. +Falco can create a child's process-table entry while parsing the parent's event, +before seeing anything from the child. The plugin must handle that lifecycle. +The numeric thread IDs (TIDs) merely let the test describe these relationships; +the important question is whether a real parser-created child gets the right +container identity, including when one of the events is missing. + +## What's in the corpus + +| Cases | Scenarios | Decision being checked | +| --- | --- | --- | +| 16 startup | Docker, CRI-O, containerd, Podman paths; host/conmon; malformed IDs; cgroup ordering | Correct identity regardless of unrelated cgroup entries; exclude host activity | +| 33 process creation | fork/clone/clone3; host, hostPID and PID-namespace containers; parent-first, child-first, missing parent or child | Identity follows a valid child through supported event orders | +| 18 focused | Exec/execveat, failed exec, ID reuse, thread clone, vfork, late discovery, installed filters, recovery and network handling | Refresh stale state and preserve downstream attribution | + +The process-creation matrix deliberately excludes parent-only PID-namespace +cases: the parent reports a namespace-local child ID, insufficient for Falco to +create the global entry. For parent-only host/hostPID cases we first assert that +Falco created a valid child with the expected cgroup, then check plugin identity. +The vfork case uses child creation, child exit, then parent return, not an +arbitrary ordering that would report an impossible lifecycle as a defect. + +`Corpus.h` holds typed startup data; `ContainerPluginReplayTest.cpp` defines the +event sequences and parameterized cases. GTest gives cases readable names so a +failure can be selected and reproduced individually. + +## What we found locally + +On parent commit `249fe750e`, **54 cases pass and 13 fail**. These failures reduce +to two root causes, not thirteen independent bugs: + +- Three cases show a later nonmatching cgroup erasing an already resolved ID. + The resulting host attribution rejects legitimate container events. +- Eight cases show missing identity for a child created from the parent's event: + six process-creation cases, one installed-filter check and one network check. + Two more cases model late-discovered processes with the same empty-cache gap. + +The network reproduction matters beyond a field assertion: with both fork events, +the production handler records one correctly attributed connection. Omit the +child event and the handler returns `IGNORED`, leaving the tracker empty. +Supplying the child event restores attribution. Separately, a host child with an +empty cached ID passes the installed filter; this does not prove delivery to Sensor. + +Across 100 shuffled local ASan/UBSan iterations (6,700 executions), every iteration +had the same 54/13 split and no sanitizer/leak diagnostics. These are behavioral +failures, not demonstrated memory-safety crashes. Supported layout controls, +complete fork orders, exec refresh, ID reuse and valid vfork ordering passed. ## Build and run locally @@ -32,6 +100,16 @@ The build reuses upstream fixture sources without enabling the entire upstream test suite. It enables Falco's TEST_INPUT engine only with this build option. Normal Collector test builds remain unchanged when the option is off. +To reproduce just the lost-connection case: + +```sh +bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/repro \ + --gtest_filter=ContainerPluginReplayTest.HostPIDConnectionTrackedWithoutChildForkEvent +``` + +`ROX_COLLECTOR_CONTAINER_PLUGIN_PATH` selects another compatible plugin module; +it does not change the linked Falco or Collector version. + ## Sanitizers Instrument C as well as C++ so libscap participates: @@ -59,6 +137,7 @@ using the same `build-builder-image` output tag and standard checkout (the merge commit for pull requests). It builds against that revision's pinned submodules. AMD64 and ARM64 each run 100 shuffled ASan/UBSan iterations. Logs, XML, and revision information are uploaded even if replay fails. +The log contains all iterations; GTest's XML describes only the final iteration. The validator needs only the builder job, not the Collector image or integration tests, and uses only read access to repository contents. @@ -66,13 +145,31 @@ The expected original-PR result is red: 54/67 cases pass and 13 fail. Known regressions are not skipped or converted to success. Fixes belong in the parent plugin branch; updating this stacked branch and rerunning provides validation. -## Limitations - -This contribution contains tests and CI only; plugin fixes belong in the parent PR. - -The tests use real Falco callbacks and production ID lookup; two also exercise -the network handler and connection tracker. The short filter-construction code -is currently duplicated from Service.cpp, not shared with service startup. -Late-import cases model the result of a successful /proc lookup because TEST_INPUT -has no live proc_get callback. There is no Sensor delivery, random mutation, or -performance benchmark here. These boundaries are detailed in CORPUS.md. +## Gaps and tradeoffs + +- **Repeatable parser tests, not live capture.** Synthetic input makes ordering + failures small and reproducible, but does not validate kernel event encoding, + real event loss, runtime discovery, deployment or Sensor delivery. Those need + separate live integration tests. +- **Real implementation, coupled fixtures.** Linking production Collector and + pinned Falco catches integration errors that a fake plugin API would miss. + It also requires a Linux builder and may need fixture changes on Falco upgrades. + Swapping a plugin file is not a comparison with an older Collector release. +- **Late discovery is modeled.** Two tests insert a valid process into Falco's + real thread manager after capture starts. TEST_INPUT has no live `/proc` lookup + callback. These support the cache-lifecycle finding, not a live-discovery claim. +- **Filter setup can drift.** The short filter construction is copied from + `Service.cpp`, rather than shared with service startup. Changes to production + filter configuration must be reflected here until a shared helper is extracted. +- **Repetition is not exploration.** Shuffling repeats the same 67 scenarios; + it does not mutate event contents, inject callback failures, explore concurrency, + or cover every process lifecycle. Bounded structured mutation is a next step, + with minimized failures promoted into named regression cases. +- **Sanitizers are not a performance test.** They cover instrumented code, not + prebuilt external libraries, and a clean run is not proof of memory safety. + Recovery of the reported CPU regression requires optimized, controlled workload + comparisons that also verify signal correctness. + +First use this corpus to validate the parent's fixes without weakening assertions. +Then extend it with structured mutation and live integration checks; keep CPU +benchmarks separate so correctness and performance results remain interpretable. From 38c75d1d9003434ee1cf2263dd75f2faebb92a76 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:25:22 -0700 Subject: [PATCH 6/8] docs: focus validator guide on usage and extension; simplify CI --- .github/workflows/plugin-validator.yml | 13 +- collector/test/plugin-replay/README.md | 258 +++++++++++-------------- 2 files changed, 111 insertions(+), 160 deletions(-) diff --git a/.github/workflows/plugin-validator.yml b/.github/workflows/plugin-validator.yml index 2a7d87c0ec..d019799ed5 100644 --- a/.github/workflows/plugin-validator.yml +++ b/.github/workflows/plugin-validator.yml @@ -13,7 +13,7 @@ permissions: jobs: replay: - name: Plugin corpus (ASan/UBSan, ${{ matrix.arch }}) + name: Plugin corpus (${{ matrix.arch }}) strategy: fail-fast: false matrix: @@ -25,11 +25,6 @@ jobs: defaults: run: shell: bash - env: - ASAN_OPTIONS: detect_leaks=1:halt_on_error=1 - UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 - REPLAY_REPEAT: '100' - REPLAY_SEED: '3939' steps: - uses: actions/checkout@v4 @@ -47,14 +42,12 @@ jobs: c++ --version } | tee artifacts/revisions.txt - - name: Configure sanitizer replay target + - name: Configure replay target run: | cmake -S . -B cmake-build \ -DBUILD_PLUGIN_REPLAY_TESTS=ON \ - -DADDRESS_SANITIZER=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DDISABLE_PROFILING=ON \ - '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer' \ 2>&1 | tee artifacts/configure.log - name: Build validator and production plugin @@ -62,7 +55,7 @@ jobs: cmake --build cmake-build --target ContainerPluginReplayTest --parallel 2 \ 2>&1 | tee artifacts/build.log - - name: Replay corpus with shuffled repetitions + - name: Replay corpus run: | bash collector/test/plugin-replay/run-corpus.sh \ "$GITHUB_WORKSPACE/cmake-build" "$GITHUB_WORKSPACE/artifacts/replay" diff --git a/collector/test/plugin-replay/README.md b/collector/test/plugin-replay/README.md index 2d66476f5c..cbe2cc8ca1 100644 --- a/collector/test/plugin-replay/README.md +++ b/collector/test/plugin-replay/README.md @@ -1,175 +1,133 @@ -# Plugin correctness validator +# Container plugin replay tests -The container plugin caches which container a process belongs to. Collector uses -that answer to exclude host activity and attribute container connections. A wrong -or missing answer can silently drop a legitimate connection or let host activity -through the filter. This validator checks those decisions before we assess speed. +Use this suite to check container attribution, host filtering and selected network +handling without starting Collector against a live kernel or Kubernetes cluster. +It supplies synthetic process events to the real Falco parser, loads the compiled +container plugin, and checks the answers through Collector's production code. -It is a 67-case deterministic regression suite, not a random fuzzer. This PR adds -tests and CI only; plugin fixes belong in the parent PR. +## Build and run -## How it works - -Each test gives a fresh Falco inspector a small process inventory and a sequence -of synthetic events, such as a process starting a child and that child connecting -to a socket. Falco's real TEST_INPUT engine parses those events and invokes the -actual compiled plugin. We check the resulting container ID, filtering decision, -and, in two tests, Collector's network handler and connection tracker. - -The path under test is: - -`scenario -> real Falco parser -> real plugin -> Collector attribution/filter -> selected network checks` - -We synthesize the input, not the plugin's answer. Tests never write the plugin's -cached container-ID field. Expected IDs are explicit scenario data, not values -computed with the plugin's own extraction logic. There is no live kernel capture, -container runtime socket, Kubernetes cluster, or Sensor service. - -Why simulate process IDs? A fork can be observed from both the parent and child. -Falco can create a child's process-table entry while parsing the parent's event, -before seeing anything from the child. The plugin must handle that lifecycle. -The numeric thread IDs (TIDs) merely let the test describe these relationships; -the important question is whether a real parser-created child gets the right -container identity, including when one of the events is missing. - -## What's in the corpus - -| Cases | Scenarios | Decision being checked | -| --- | --- | --- | -| 16 startup | Docker, CRI-O, containerd, Podman paths; host/conmon; malformed IDs; cgroup ordering | Correct identity regardless of unrelated cgroup entries; exclude host activity | -| 33 process creation | fork/clone/clone3; host, hostPID and PID-namespace containers; parent-first, child-first, missing parent or child | Identity follows a valid child through supported event orders | -| 18 focused | Exec/execveat, failed exec, ID reuse, thread clone, vfork, late discovery, installed filters, recovery and network handling | Refresh stale state and preserve downstream attribution | - -The process-creation matrix deliberately excludes parent-only PID-namespace -cases: the parent reports a namespace-local child ID, insufficient for Falco to -create the global entry. For parent-only host/hostPID cases we first assert that -Falco created a valid child with the expected cgroup, then check plugin identity. -The vfork case uses child creation, child exit, then parent return, not an -arbitrary ordering that would report an impossible lifecycle as a defect. - -`Corpus.h` holds typed startup data; `ContainerPluginReplayTest.cpp` defines the -event sequences and parameterized cases. GTest gives cases readable names so a -failure can be selected and reproduced individually. - -## What we found locally - -On parent commit `249fe750e`, **54 cases pass and 13 fail**. These failures reduce -to two root causes, not thirteen independent bugs: - -- Three cases show a later nonmatching cgroup erasing an already resolved ID. - The resulting host attribution rejects legitimate container events. -- Eight cases show missing identity for a child created from the parent's event: - six process-creation cases, one installed-filter check and one network check. - Two more cases model late-discovered processes with the same empty-cache gap. - -The network reproduction matters beyond a field assertion: with both fork events, -the production handler records one correctly attributed connection. Omit the -child event and the handler returns `IGNORED`, leaving the tracker empty. -Supplying the child event restores attribution. Separately, a host child with an -empty cached ID passes the installed filter; this does not prove delivery to Sensor. - -Across 100 shuffled local ASan/UBSan iterations (6,700 executions), every iteration -had the same 54/13 split and no sanitizer/leak diagnostics. These are behavioral -failures, not demonstrated memory-safety crashes. Supported layout controls, -complete fork orders, exec refresh, ID reuse and valid vfork ordering passed. - -## Build and run locally - -In a Linux Collector builder environment with source at `/src`: +Run inside a Linux Collector builder environment, with the repository at `/src`. +Initialize the required submodules first. The source tree must be writable because +Falco generates some headers there during the build. ```sh +cd /src +git submodule update --init falcosecurity-libs collector/proto/third_party/stackrox cmake -S /src -B /build \ -DBUILD_PLUGIN_REPLAY_TESTS=ON \ -DCMAKE_BUILD_TYPE=Debug -DDISABLE_PROFILING=ON -cmake --build /build --target ContainerPluginReplayTest -j4 +cmake --build /build --target ContainerPluginReplayTest -j2 bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/plugin-results ``` -Initialize the pinned `falcosecurity-libs` and -`collector/proto/third_party/stackrox` submodules first. Falco sources must be -writable because its build generates some source-tree headers. +The runner writes `run.log` and `results.xml` and returns nonzero if any assertion +fails. It sets the plugin path automatically. To select or list cases, pass GTest +arguments after the two directory arguments: -The runner retains logs and XML and returns nonzero on test failure. CTest also -registers `ContainerPluginReplayTest` and supplies the plugin path automatically: +```sh +bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/plugin-results \ + --gtest_filter='*MatchingCgroup*' +bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/plugin-results \ + --gtest_list_tests +``` + +CTest also registers the target and supplies its plugin path: ```sh ctest --test-dir /build -R '^ContainerPluginReplayTest$' --output-on-failure ``` -The build reuses upstream fixture sources without enabling the entire upstream -test suite. It enables Falco's TEST_INPUT engine only with this build option. -Normal Collector test builds remain unchanged when the option is off. +For order-dependent failures, set `REPLAY_REPEAT=100 REPLAY_SEED=3939` before the +runner command. This repeats and shuffles the same cases, not their event contents. +The log retains every iteration; the XML describes only the last iteration. +`ROX_COLLECTOR_CONTAINER_PLUGIN_PATH` can select a compatible plugin module, but +does not change the linked Falco or Collector version. -To reproduce just the lost-connection case: +## How a test works -```sh -bash /src/collector/test/plugin-replay/run-corpus.sh /build /tmp/repro \ - --gtest_filter=ContainerPluginReplayTest.HostPIDConnectionTrackedWithoutChildForkEvent -``` +Each case starts with a fresh inspector and plugin. `SeedThread` supplies the +initial process inventory and cgroups. `Open` starts the inspector, letting Falco +invoke the plugin's capture callback. Event helpers then feed synthetic events +through Falco's TEST_INPUT engine; Falco performs parsing, process-table updates +and plugin callbacks. Tests do not call those callbacks directly or populate the +plugin's cached container-ID field. -`ROX_COLLECTOR_CONTAINER_PLUGIN_PATH` selects another compatible plugin module; -it does not change the linked Falco or Collector version. +`ExpectAttribution` checks Collector's container ID and the plugin-backed filter. +Network-focused cases additionally exercise `NetworkSignalHandler` and +`ConnectionTracker`. Expected IDs are explicit test data, never calculated with +the plugin's extraction function. -## Sanitizers +Process IDs simply connect events to their parent or child. For example, Falco +may learn about a child from the parent's fork event before observing the child's +event. Keeping those inputs separate lets a test check attribution when events +arrive in a different order or one is missing. -Instrument C as well as C++ so libscap participates: +## Extend the corpus -```sh -cmake -S /src -B /build-asan \ - -DBUILD_PLUGIN_REPLAY_TESTS=ON -DADDRESS_SANITIZER=ON \ - -DCMAKE_BUILD_TYPE=Debug -DDISABLE_PROFILING=ON \ - '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer' -cmake --build /build-asan --target ContainerPluginReplayTest -j2 -REPLAY_REPEAT=100 REPLAY_SEED=3939 \ -ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \ -UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ - bash /src/collector/test/plugin-replay/run-corpus.sh /build-asan /tmp/plugin-asan +The corpus has three groups: + +- Startup layouts in `Corpus.h`: Docker, CRI-O, containerd and Podman cgroups, + host/conmon exclusion, malformed IDs and cgroup ordering. +- Process-creation combinations in `Corpus.h`: fork/clone/clone3, host or container + origins, and parent/child event ordering. The parameterized test supplies events. +- Focused `TEST_F` cases in `ContainerPluginReplayTest.cpp`: exec refresh, process-ID + reuse, thread clone, vfork, late discovery, filtering and connection tracking. + +### Add a cgroup layout + +Add a named `StartupCase` to `StartupCases()` with controller-prefixed cgroup +strings and an explicit expected short ID (or `""` for host/excluded activity). +For example, a new ordering case could be: + +```cpp +{"HostThenDocker", {"cpuset=/", "memory=/docker/" + kA}, "aaaaaaaaaaaa"}, ``` -External prebuilt builder libraries are not rebuilt with instrumentation. If a -Debug build already generated the BPF skeleton, the sanitizer build can reuse it -with `-DMODERN_BPF_SKEL_DIR=/build/skel_dir`. No BPF program is loaded by replay. - -## CI - -Main CI calls `.github/workflows/plugin-validator.yml` alongside unit tests, -using the same `build-builder-image` output tag and standard checkout (the merge -commit for pull requests). It builds against that revision's pinned submodules. -AMD64 and ARM64 each run 100 shuffled ASan/UBSan -iterations. Logs, XML, and revision information are uploaded even if replay fails. -The log contains all iterations; GTest's XML describes only the final iteration. -The validator needs only the builder job, not the Collector image or integration -tests, and uses only read access to repository contents. - -The expected original-PR result is red: 54/67 cases pass and 13 fail. Known -regressions are not skipped or converted to success. Fixes belong in the parent -plugin branch; updating this stacked branch and rerunning provides validation. - -## Gaps and tradeoffs - -- **Repeatable parser tests, not live capture.** Synthetic input makes ordering - failures small and reproducible, but does not validate kernel event encoding, - real event loss, runtime discovery, deployment or Sensor delivery. Those need - separate live integration tests. -- **Real implementation, coupled fixtures.** Linking production Collector and - pinned Falco catches integration errors that a fake plugin API would miss. - It also requires a Linux builder and may need fixture changes on Falco upgrades. - Swapping a plugin file is not a comparison with an older Collector release. -- **Late discovery is modeled.** Two tests insert a valid process into Falco's - real thread manager after capture starts. TEST_INPUT has no live `/proc` lookup - callback. These support the cache-lifecycle finding, not a live-discovery claim. -- **Filter setup can drift.** The short filter construction is copied from - `Service.cpp`, rather than shared with service startup. Changes to production - filter configuration must be reflected here until a shared helper is extracted. -- **Repetition is not exploration.** Shuffling repeats the same 67 scenarios; - it does not mutate event contents, inject callback failures, explore concurrency, - or cover every process lifecycle. Bounded structured mutation is a next step, - with minimized failures promoted into named regression cases. -- **Sanitizers are not a performance test.** They cover instrumented code, not - prebuilt external libraries, and a clean run is not proof of memory safety. - Recovery of the reported CPU regression requires optimized, controlled workload - comparisons that also verify signal correctness. - -First use this corpus to validate the parent's fixes without weakening assertions. -Then extend it with structured mutation and live integration checks; keep CPU -benchmarks separate so correctness and performance results remain interpretable. +The existing parameterized test seeds the process and checks attribution and +filtering. Use a unique descriptive name so the case is easy to select in GTest. + +### Add a lifecycle scenario + +Add a `TEST_F(ContainerPluginReplayTest, DescriptiveName)` in the replay test file: + +1. Seed only the processes known before capture, then call `Open()`. +2. Generate the smallest valid event sequence needed for the scenario, using the + existing Falco helpers. Keep timestamps increasing and parent/child IDs coherent. +3. Assert prerequisites such as the child's existence and cgroups before checking + attribution. A missing parser-created process is different from a plugin bug. +4. Check the expected ID and filtering decision; use the network-handler helper + when the requirement is that a connection actually reaches the tracker. +5. Include a nearby positive control when omitting or reordering an event, and + run the focused case followed by the full corpus. + +Extend `ForkCases()` only when an event has the same encoding and expectations as +the existing parameterized test. Use a focused case for a different lifecycle. +Do not remove its parent-only PID-namespace exclusion: that event supplies a +namespace-local child ID, insufficient to create the global child entry. Similarly, +vfork sequences must respect the child's exit before the parent's return. + +## CI and maintenance + +Main CI calls `.github/workflows/plugin-validator.yml` alongside unit tests using +the same builder-tag output. It builds the standard checkout and pinned submodules +on AMD64 and ARM64, runs the corpus once, and uploads logs, XML and build/revision +information even on failure. Assertions are not skipped or converted to success. + +`BUILD_PLUGIN_REPLAY_TESTS` is opt-in. Its CMake target compiles the pinned Falco +test helpers and enables TEST_INPUT without enabling the entire upstream suite. +Normal builds are unchanged when the option is off. When updating Falco, check +helper signatures and event semantics as well as whether the target still builds. + +## Boundaries to preserve + +- Synthetic input does not test live kernel capture, actual event loss, runtime + discovery, deployment or Sensor delivery. Add live integration tests for those. +- `ImportLateThread` models the result of a successful `/proc` lookup by inserting + a valid process into Falco's thread manager. TEST_INPUT has no live lookup callback; + keep those tests clearly distinguished from event-only reproductions. +- Filter construction is currently copied from `Service.cpp`. Keep it aligned + with production configuration until a shared helper replaces the duplication. +- Replays are deterministic correctness tests, not random fuzzing or CPU benchmarks. + If adding structured mutation, retain reproducible seeds and promote minimized + failures to named cases. Measure performance separately with controlled workloads. From b04a862b67f663445e18aa47c6ef5e3b0295cbff Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:32:02 -0700 Subject: [PATCH 7/8] docs: outline planned validator extensions --- collector/test/plugin-replay/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/collector/test/plugin-replay/README.md b/collector/test/plugin-replay/README.md index cbe2cc8ca1..6ca881251f 100644 --- a/collector/test/plugin-replay/README.md +++ b/collector/test/plugin-replay/README.md @@ -131,3 +131,21 @@ helper signatures and event semantics as well as whether the target still builds - Replays are deterministic correctness tests, not random fuzzing or CPU benchmarks. If adding structured mutation, retain reproducible seeds and promote minimized failures to named cases. Measure performance separately with controlled workloads. + +## Next steps + +Planned extensions, not currently supported: + +- Expand the corpus with cgroup changes after startup, plugin restart/reinitialization, + and callback read/write failures. Keep failure injection separate from valid + event-sequence tests. +- Add bounded structured fuzzing of valid event sequences. Save reproducible seeds + and promote minimized failures into named regression cases. +- Extend replay first to the post-upgrade Collector without the plugin, then to + the pre-upgrade Collector. Reuse scenario expectations, but build each revision + with its own pinned Falco dependency and event/attribution adapter; older output + is a comparison, not the correctness oracle. +- Add packaged-image integration tests for live discovery, event loss and signal + delivery. Keep controlled CPU benchmarks separate from correctness replay. + +Link these items to tracking issues as the work is scoped. From 2704e8aadd1dfca3e77f8e4ba5fb02fcaed8ebd2 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 8 Sep 2026 16:44:13 -0700 Subject: [PATCH 8/8] ci: validate plugin with ASan and UBSan --- .github/workflows/plugin-validator.yml | 9 +++++++-- collector/test/plugin-replay/README.md | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/plugin-validator.yml b/.github/workflows/plugin-validator.yml index d019799ed5..ba3e5baa14 100644 --- a/.github/workflows/plugin-validator.yml +++ b/.github/workflows/plugin-validator.yml @@ -13,7 +13,7 @@ permissions: jobs: replay: - name: Plugin corpus (${{ matrix.arch }}) + name: Plugin corpus (ASan/UBSan, ${{ matrix.arch }}) strategy: fail-fast: false matrix: @@ -25,6 +25,9 @@ jobs: defaults: run: shell: bash + env: + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 steps: - uses: actions/checkout@v4 @@ -42,12 +45,14 @@ jobs: c++ --version } | tee artifacts/revisions.txt - - name: Configure replay target + - name: Configure sanitizer replay target run: | cmake -S . -B cmake-build \ -DBUILD_PLUGIN_REPLAY_TESTS=ON \ + -DADDRESS_SANITIZER=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DDISABLE_PROFILING=ON \ + '-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer' \ 2>&1 | tee artifacts/configure.log - name: Build validator and production plugin diff --git a/collector/test/plugin-replay/README.md b/collector/test/plugin-replay/README.md index 6ca881251f..6bba5aa1d5 100644 --- a/collector/test/plugin-replay/README.md +++ b/collector/test/plugin-replay/README.md @@ -111,9 +111,17 @@ vfork sequences must respect the child's exit before the parent's return. Main CI calls `.github/workflows/plugin-validator.yml` alongside unit tests using the same builder-tag output. It builds the standard checkout and pinned submodules -on AMD64 and ARM64, runs the corpus once, and uploads logs, XML and build/revision +on AMD64 and ARM64 with ASan/UBSan, runs the corpus once, and uploads logs, XML and build/revision information even on failure. Assertions are not skipped or converted to success. +To match CI locally, add `-DADDRESS_SANITIZER=ON` and +`'-DCMAKE_C_FLAGS=-fsanitize=address,undefined -fno-omit-frame-pointer'` to the +Debug configure command in a separate build directory. The repository's +`ADDRESS_SANITIZER` option enables both sanitizers for C++; the C flags also +instrument libscap. Run with `ASAN_OPTIONS=detect_leaks=1:halt_on_error=1` and +`UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1`. Prebuilt external libraries +are not rebuilt with instrumentation. Valgrind is not used by this workflow. + `BUILD_PLUGIN_REPLAY_TESTS` is opt-in. Its CMake target compiles the pinned Falco test helpers and enables TEST_INPUT without enabling the entire upstream suite. Normal builds are unchanged when the option is off. When updating Falco, check