diff --git a/engine/include/backend/engine/evaluator/knn.hpp b/engine/include/backend/engine/evaluator/knn.hpp index 2c3664b..b921f69 100644 --- a/engine/include/backend/engine/evaluator/knn.hpp +++ b/engine/include/backend/engine/evaluator/knn.hpp @@ -10,10 +10,9 @@ namespace evaluator { struct KnnScores { - double kendalls_tau = std::numeric_limits::quiet_NaN(); double recall_at_k = std::numeric_limits::quiet_NaN(); + double kendalls_tau_topk = std::numeric_limits::quiet_NaN(); std::size_t query_count = 0; - std::size_t tau_query_count = 0; // queries with at least two shared neighbors }; // Each score is averaged over the queries where it is defined, NaN if it is defined for none. @@ -23,9 +22,10 @@ KnnScores evaluate_knn(const core::TrajectoryDb &compressed, const query::KnnWor namespace evaluator::knn { -// Rank agreement on shared neighbors only; NaN when fewer than two are shared. -double kendalls_tau(const std::vector &original, - const std::vector &simplified); +// Fagin's K^(p) over the union of both lists as a similarity in [0, 1], 1 = identical. A pair +// that one list holds neither item of costs p. NaN if no pair can be penalized (e.g. both empty). +double kendalls_tau_topk(const std::vector &original, + const std::vector &simplified, double p); // Share of original that also appears in simplified, ignoring order. NaN if original is empty. double recall_at_k(const std::vector &original, diff --git a/engine/include/backend/engine/query/knn.hpp b/engine/include/backend/engine/query/knn.hpp index 9116d9b..87554ad 100644 --- a/engine/include/backend/engine/query/knn.hpp +++ b/engine/include/backend/engine/query/knn.hpp @@ -16,12 +16,14 @@ struct KnnQuery { struct KnnConfig { std::size_t k = 5; std::size_t query_count = 10; // 0 = use every trajectory in the query pool + double p = 0.5; // top-k tau penalty for a pair one list holds neither of, in [0, 1] }; struct KnnWorkload { core::TrajectoryDb queries; std::vector entries; std::size_t k; + double p = 0.5; }; // Samples query_count trajectories from query_pool (the held-out split) and stores their k diff --git a/engine/src/evaluator/knn.cpp b/engine/src/evaluator/knn.cpp index 2cd1a41..fb1d25e 100644 --- a/engine/src/evaluator/knn.cpp +++ b/engine/src/evaluator/knn.cpp @@ -1,5 +1,6 @@ #include "backend/engine/evaluator/knn.hpp" #include "backend/engine/core/parallel.hpp" +#include #include #include #include @@ -22,27 +23,57 @@ shared_ranks(const std::vector &original, const std::vector positions_in(const std::vector &ranking, + const std::vector &full) { + const std::size_t absent = ranking.size(); + std::vector positions(full.size(), absent); + for (std::size_t i = 0; i < full.size(); i++) { + for (std::size_t j = 0; j < ranking.size(); j++) { + if (ranking[j] == full[i]) { + positions[i] = j; + break; + } + } + } + return positions; +} + } // namespace -double kendalls_tau(const std::vector &original, - const std::vector &simplified) { - std::vector> overlap = shared_ranks(original, simplified); - if (overlap.size() < 2) { - return std::numeric_limits::quiet_NaN(); +// Fagin et al., "Comparing top k lists" (2003) +double kendalls_tau_topk(const std::vector &original, + const std::vector &simplified, double p) { + + // the lists are in rank order, not sorted, so std::set_union does not apply + std::vector union_vec = original; + for (std::size_t id : simplified) { + if (std::find(original.begin(), original.end(), id) == original.end()) { + union_vec.push_back(id); + } } - std::size_t concordant = 0, discordant = 0; - for (std::size_t i = 0; i < overlap.size(); i++) { - for (std::size_t j = i + 1; j < overlap.size(); j++) { - if (overlap[i].second < overlap[j].second) { - concordant++; - } else { - discordant++; + const auto original_rank = positions_in(original, union_vec); + const auto simplified_rank = positions_in(simplified, union_vec); + + double distance = 0.0; + for (std::size_t i = 0; i < union_vec.size(); i++) { + for (std::size_t j = i + 1; j < union_vec.size(); j++) { + if (original_rank[i] == original_rank[j] || simplified_rank[i] == simplified_rank[j]) { + distance += p; // one list holds neither item, so it cannot tell their order + } else if ((original_rank[i] < original_rank[j]) != + (simplified_rank[i] < simplified_rank[j])) { + distance += 1.0; // the lists order the pair in opposite directions } } } - return (static_cast(concordant) - static_cast(discordant)) / - static_cast(concordant + discordant); + + const double original_size = static_cast(original.size()); + const double simplified_size = static_cast(simplified.size()); + const double max_distance = + original_size * simplified_size + + p * (original_size * (original_size - 1) / 2 + simplified_size * (simplified_size - 1) / 2); + return 1.0 - distance / max_distance; } double recall_at_k(const std::vector &original, @@ -59,24 +90,24 @@ double recall_at_k(const std::vector &original, namespace evaluator { KnnScores evaluate_knn(const core::TrajectoryDb &compressed, const query::KnnWorkload &workload) { - double tau_sum = 0.0; - std::size_t tau_scored = 0; double recall_sum = 0.0; std::size_t recall_scored = 0; - std::vector taus(workload.entries.size()); + double topk_tau_sum = 0.0; + std::size_t topk_tau_scored = 0; std::vector recalls(workload.entries.size()); + std::vector topk_taus(workload.entries.size()); core::parallel_for(workload.entries.size(), [&](std::size_t i) { const auto &query = workload.entries[i]; const auto trajectory = workload.queries.trajectory(query.query_index); const auto simplified = query::knn(compressed, trajectory, workload.k); - taus[i] = knn::kendalls_tau(query.reference, simplified); recalls[i] = knn::recall_at_k(query.reference, simplified); + topk_taus[i] = knn::kendalls_tau_topk(query.reference, simplified, workload.p); }); for (std::size_t i = 0; i < workload.entries.size(); ++i) { - const double tau = taus[i]; - if (!std::isnan(tau)) { - tau_sum += tau; - ++tau_scored; + const double topk_tau = topk_taus[i]; + if (!std::isnan(topk_tau)) { + topk_tau_sum += topk_tau; + ++topk_tau_scored; } const double recall = recalls[i]; if (!std::isnan(recall)) { @@ -87,11 +118,10 @@ KnnScores evaluate_knn(const core::TrajectoryDb &compressed, const query::KnnWor KnnScores scores; scores.query_count = workload.entries.size(); - scores.tau_query_count = tau_scored; - if (tau_scored != 0) - scores.kendalls_tau = tau_sum / static_cast(tau_scored); if (recall_scored != 0) scores.recall_at_k = recall_sum / static_cast(recall_scored); + if (topk_tau_scored != 0) + scores.kendalls_tau_topk = topk_tau_sum / static_cast(topk_tau_scored); return scores; } diff --git a/engine/src/pipeline/config.cpp b/engine/src/pipeline/config.cpp index e62040b..7edad0e 100644 --- a/engine/src/pipeline/config.cpp +++ b/engine/src/pipeline/config.cpp @@ -45,9 +45,10 @@ void parse_split_config(Config &config, const json &split) { } void parse_knn_config(Config &config, const json &knn) { - validate_keys(knn, {"k", "query_count"}, "knn"); + validate_keys(knn, {"k", "query_count", "p"}, "knn"); config.knn.k = knn.value("k", config.knn.k); config.knn.query_count = knn.value("query_count", config.knn.query_count); + config.knn.p = knn.value("p", config.knn.p); } void parse_range_config(Config &config, const json &range) { diff --git a/engine/src/pipeline/pipeline.cpp b/engine/src/pipeline/pipeline.cpp index a50befd..24b9695 100644 --- a/engine/src/pipeline/pipeline.cpp +++ b/engine/src/pipeline/pipeline.cpp @@ -45,9 +45,9 @@ std::size_t total_budget(const core::TrajectoryDb &trajectories, double share) { } void print_scores(const evaluator::Scoretable &scores) { - std::cout << " knn tau on shared neighbors: " << scores.knn.kendalls_tau << " (" - << scores.knn.tau_query_count << '/' << scores.knn.query_count << " queries)" - << ", recall@k: " << scores.knn.recall_at_k << '\n'; + std::cout << " knn top-k tau: " << scores.knn.kendalls_tau_topk + << ", recall@k: " << scores.knn.recall_at_k << " (" << scores.knn.query_count + << " queries)\n"; std::cout << " where-at: within radius=" << scores.where_at.within_radius << ", average hit=" << scores.where_at.average_hit << '\n'; std::cout << " range recall: " << scores.range_recall << '\n'; diff --git a/engine/src/query/knn.cpp b/engine/src/query/knn.cpp index 6c8d414..90d9339 100644 --- a/engine/src/query/knn.cpp +++ b/engine/src/query/knn.cpp @@ -275,6 +275,9 @@ KnnWorkload generate_knn_queries(const core::TrajectoryDb &original, if (config.k == 0) { throw std::invalid_argument("knn: k = 0, no neighbors"); } + if (config.p < 0.0 || config.p > 1.0) { + throw std::invalid_argument("knn: p must be in [0, 1]"); + } std::vector picked(query_pool.size()); std::ranges::iota(picked.begin(), picked.end(), 0); @@ -284,7 +287,7 @@ KnnWorkload generate_knn_queries(const core::TrajectoryDb &original, picked.resize(config.query_count); } - KnnWorkload results{core::subset(query_pool, picked), {}, config.k}; + KnnWorkload results{core::subset(query_pool, picked), {}, config.k, config.p}; results.entries.resize(results.queries.size()); core::parallel_for(results.queries.size(), [&](std::size_t i) { diff --git a/engine/tests/config_test.cpp b/engine/tests/config_test.cpp index d38a053..da924e6 100644 --- a/engine/tests/config_test.cpp +++ b/engine/tests/config_test.cpp @@ -41,7 +41,7 @@ TEST(Config, ReadsTopLevelAndNestedKeys) { "split": {"type": "Time", "query_percentage": 0.3}, "budget_shares": [0.5], "algorithms": ["SED"], - "knn": {"k": 3, "query_count": 0}, + "knn": {"k": 3, "query_count": 0, "p": 0.25}, "range": {"query_count": 10, "box_size_m": 500, "window_seconds": 3600}, "where_at": {"probes_per_trajectory": 16, "max_gap_seconds": 30}, "where_at_scoring": {"hit_radius_m": 50, "max_tolerance_m": 300} @@ -53,6 +53,7 @@ TEST(Config, ReadsTopLevelAndNestedKeys) { EXPECT_EQ(config.algorithms, std::vector{selection::AlgorithmType::SED}); EXPECT_EQ(config.knn.k, 3); EXPECT_EQ(config.knn.query_count, 0); + EXPECT_DOUBLE_EQ(config.knn.p, 0.25); EXPECT_EQ(config.range.query_count, 10); EXPECT_DOUBLE_EQ(config.range.box_size_m, 500); EXPECT_EQ(config.range.window_seconds, 3600); diff --git a/engine/tests/evaluator_test.cpp b/engine/tests/evaluator_test.cpp index 8600451..46dfef5 100644 --- a/engine/tests/evaluator_test.cpp +++ b/engine/tests/evaluator_test.cpp @@ -29,7 +29,7 @@ TEST(Evaluator, CollectsBothWhereAtScoresWithoutUnconfiguredQueries) { const auto scores = evaluator.evaluate(); EXPECT_DOUBLE_EQ(scores.where_at.within_radius, 1); EXPECT_DOUBLE_EQ(scores.where_at.average_hit, 1); - EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau)); + EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau_topk)); EXPECT_TRUE(std::isnan(scores.knn.recall_at_k)); EXPECT_TRUE(std::isnan(scores.range_recall)); } @@ -59,7 +59,7 @@ TEST(Evaluator, NoWorkloadsReturnsUndefinedScores) { const auto db = original(); evaluator::Evaluator evaluator(db, db); const auto scores = evaluator.evaluate(); - EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau)); + EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau_topk)); EXPECT_TRUE(std::isnan(scores.knn.recall_at_k)); EXPECT_TRUE(std::isnan(scores.range_recall)); EXPECT_TRUE(std::isnan(scores.where_at.within_radius)); @@ -98,33 +98,41 @@ TEST(Evaluator, CollectsKnnAndWhereAtScoresTogether) { evaluator::Evaluator evaluator(db, db, {.knn = &knn, .where_at = where_at.queries}); const auto scores = evaluator.evaluate(); - EXPECT_DOUBLE_EQ(scores.knn.kendalls_tau, 1); EXPECT_DOUBLE_EQ(scores.knn.recall_at_k, 1); + EXPECT_DOUBLE_EQ(scores.knn.kendalls_tau_topk, 1); EXPECT_EQ(scores.knn.query_count, 2u); - EXPECT_EQ(scores.knn.tau_query_count, 2u); EXPECT_DOUBLE_EQ(scores.where_at.within_radius, 1); EXPECT_DOUBLE_EQ(scores.where_at.average_hit, 1); - EXPECT_DOUBLE_EQ(evaluator.get_knn_scores().kendalls_tau, scores.knn.kendalls_tau); + EXPECT_DOUBLE_EQ(evaluator.get_knn_scores().kendalls_tau_topk, scores.knn.kendalls_tau_topk); EXPECT_DOUBLE_EQ(evaluator.get_knn_scores().recall_at_k, scores.knn.recall_at_k); } -TEST(KnnScores, ReportsCoverageWhenOnlySomeQueriesShareEnoughNeighbors) { +TEST(KnnScores, AveragesOverQueriesWithPartialOverlap) { const core::TrajectoryDb db({0, 2, 4, 6, 8}, {10, 20, 30, 40}, {0, 10, 0, 10, 0, 10, 0, 10}, std::vector(8, 39.95), {116.35, 116.35, 116.36, 116.36, 116.37, 116.37, 116.38, 116.38}); const query::KnnWorkload workload{db, {{0, {0, 1}}, {0, {0, 2}}, {0, {2, 3}}}, 2}; const auto scores = evaluator::evaluate_knn(db, workload); EXPECT_EQ(scores.query_count, 3u); - EXPECT_EQ(scores.tau_query_count, 1u); - EXPECT_DOUBLE_EQ(scores.kendalls_tau, 1.0); + // the compressed answer is {0, 1}: identical, one swap-in (4/5 of max), disjoint + EXPECT_DOUBLE_EQ(scores.kendalls_tau_topk, (1.0 + 0.8 + 0.0) / 3); EXPECT_DOUBLE_EQ(scores.recall_at_k, 0.5); } +TEST(KnnScores, TopkTauScoresPairsOutsideTheOverlap) { + // (7,3) is swapped, and 9 and 5 each only appear in one list: 2 of a possible 9 + 0.5 * 6 + EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({7, 3, 9}, {3, 7, 5}, 0.5), 1.0 - 2.0 / 12); + EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({1, 2, 3}, {1, 2, 3}, 0.5), 1.0); + EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({1, 2}, {3, 4}, 0.5), 0.0); + // the simplified list holds neither 2 nor 3, so (2,3) costs p + EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({1, 2, 3}, {1}, 1.0), 1.0 - 1.0 / 6); + EXPECT_TRUE(std::isnan(evaluator::knn::kendalls_tau_topk({}, {}, 0.5))); +} + TEST(KnnScores, EmptyWorkloadHasZeroCoverageAndUndefinedScores) { const auto db = original(); const auto scores = evaluator::evaluate_knn(db, {db, {}, 2}); EXPECT_EQ(scores.query_count, 0u); - EXPECT_EQ(scores.tau_query_count, 0u); - EXPECT_TRUE(std::isnan(scores.kendalls_tau)); + EXPECT_TRUE(std::isnan(scores.kendalls_tau_topk)); EXPECT_TRUE(std::isnan(scores.recall_at_k)); }