improved kendalls tau with compare top k lists

engine/include/backend/engine/evaluator/knn.hpp

@@ -10,10 +10,9 @@
 namespace evaluator {
 
 struct KnnScores {
-    double kendalls_tau = std::numeric_limits<double>::quiet_NaN();
     double recall_at_k = std::numeric_limits<double>::quiet_NaN();
+    double kendalls_tau_topk = std::numeric_limits<double>::quiet_NaN();
     std::size_t query_count = 0;
-    std::size_t tau_query_count = 0; // queries with at least two shared neighbors
 };
 
 // Each score is averaged over the queries where it is defined, NaN if it is defined for none.
@@ -23,9 +22,10 @@ KnnScores evaluate_knn(const core::TrajectoryDb &compressed, const query::KnnWor
 
 namespace evaluator::knn {
 
-// Rank agreement on shared neighbors only; NaN when fewer than two are shared.
-double kendalls_tau(const std::vector<std::size_t> &original,
-                    const std::vector<std::size_t> &simplified);
+// Fagin's K^(p) over the union of both lists as a similarity in [0, 1], 1 = identical. A pair
+// that one list holds neither item of costs p. NaN if no pair can be penalized (e.g. both empty).
+double kendalls_tau_topk(const std::vector<std::size_t> &original,
+                         const std::vector<std::size_t> &simplified, double p);
 
 // Share of original that also appears in simplified, ignoring order. NaN if original is empty.
 double recall_at_k(const std::vector<std::size_t> &original,

engine/include/backend/engine/query/knn.hpp

@@ -16,12 +16,14 @@ struct KnnQuery {
 struct KnnConfig {
     std::size_t k = 5;
     std::size_t query_count = 10; // 0 = use every trajectory in the query pool
+    double p = 0.5; // top-k tau penalty for a pair one list holds neither of, in [0, 1]
 };
 
 struct KnnWorkload {
     core::TrajectoryDb queries;
     std::vector<KnnQuery> entries;
     std::size_t k;
+    double p = 0.5;
 };
 
 // Samples query_count trajectories from query_pool (the held-out split) and stores their k

engine/src/evaluator/knn.cpp

@@ -1,5 +1,6 @@
 #include "backend/engine/evaluator/knn.hpp"
 #include "backend/engine/core/parallel.hpp"
+#include <algorithm>
 #include <cmath>
 #include <cstddef>
 #include <limits>
@@ -22,27 +23,57 @@ shared_ranks(const std::vector<std::size_t> &original, const std::vector<std::si
     }
     return shared;
 }
+
+std::vector<std::size_t> positions_in(const std::vector<std::size_t> &ranking,
+                                      const std::vector<std::size_t> &full) {
+    const std::size_t absent = ranking.size();
+    std::vector<std::size_t> positions(full.size(), absent);
+    for (std::size_t i = 0; i < full.size(); i++) {
+        for (std::size_t j = 0; j < ranking.size(); j++) {
+            if (ranking[j] == full[i]) {
+                positions[i] = j;
+                break;
+            }
+        }
+    }
+    return positions;
+}
+
 } // namespace
 
-double kendalls_tau(const std::vector<std::size_t> &original,
-                    const std::vector<std::size_t> &simplified) {
-    std::vector<std::pair<std::size_t, std::size_t>> overlap = shared_ranks(original, simplified);
-    if (overlap.size() < 2) {
-        return std::numeric_limits<double>::quiet_NaN();
+// Fagin et al., "Comparing top k lists" (2003)
+double kendalls_tau_topk(const std::vector<std::size_t> &original,
+                         const std::vector<std::size_t> &simplified, double p) {
+
+    // the lists are in rank order, not sorted, so std::set_union does not apply
+    std::vector<std::size_t> union_vec = original;
+    for (std::size_t id : simplified) {
+        if (std::find(original.begin(), original.end(), id) == original.end()) {
+            union_vec.push_back(id);
+        }
     }
-    std::size_t concordant = 0, discordant = 0;
 
-    for (std::size_t i = 0; i < overlap.size(); i++) {
-        for (std::size_t j = i + 1; j < overlap.size(); j++) {
-            if (overlap[i].second < overlap[j].second) {
-                concordant++;
-            } else {
-                discordant++;
+    const auto original_rank = positions_in(original, union_vec);
+    const auto simplified_rank = positions_in(simplified, union_vec);
+
+    double distance = 0.0;
+    for (std::size_t i = 0; i < union_vec.size(); i++) {
+        for (std::size_t j = i + 1; j < union_vec.size(); j++) {
+            if (original_rank[i] == original_rank[j] || simplified_rank[i] == simplified_rank[j]) {
+                distance += p; // one list holds neither item, so it cannot tell their order
+            } else if ((original_rank[i] < original_rank[j]) !=
+                       (simplified_rank[i] < simplified_rank[j])) {
+                distance += 1.0; // the lists order the pair in opposite directions
             }
         }
     }
-    return (static_cast<double>(concordant) - static_cast<double>(discordant)) /
-           static_cast<double>(concordant + discordant);
+
+    const double original_size = static_cast<double>(original.size());
+    const double simplified_size = static_cast<double>(simplified.size());
+    const double max_distance =
+        original_size * simplified_size +
+        p * (original_size * (original_size - 1) / 2 + simplified_size * (simplified_size - 1) / 2);
+    return 1.0 - distance / max_distance;
 }
 
 double recall_at_k(const std::vector<std::size_t> &original,
@@ -59,24 +90,24 @@ double recall_at_k(const std::vector<std::size_t> &original,
 namespace evaluator {
 
 KnnScores evaluate_knn(const core::TrajectoryDb &compressed, const query::KnnWorkload &workload) {
-    double tau_sum = 0.0;
-    std::size_t tau_scored = 0;
     double recall_sum = 0.0;
     std::size_t recall_scored = 0;
-    std::vector<double> taus(workload.entries.size());
+    double topk_tau_sum = 0.0;
+    std::size_t topk_tau_scored = 0;
     std::vector<double> recalls(workload.entries.size());
+    std::vector<double> topk_taus(workload.entries.size());
     core::parallel_for(workload.entries.size(), [&](std::size_t i) {
         const auto &query = workload.entries[i];
         const auto trajectory = workload.queries.trajectory(query.query_index);
         const auto simplified = query::knn(compressed, trajectory, workload.k);
-        taus[i] = knn::kendalls_tau(query.reference, simplified);
         recalls[i] = knn::recall_at_k(query.reference, simplified);
+        topk_taus[i] = knn::kendalls_tau_topk(query.reference, simplified, workload.p);
     });
     for (std::size_t i = 0; i < workload.entries.size(); ++i) {
-        const double tau = taus[i];
-        if (!std::isnan(tau)) {
-            tau_sum += tau;
-            ++tau_scored;
+        const double topk_tau = topk_taus[i];
+        if (!std::isnan(topk_tau)) {
+            topk_tau_sum += topk_tau;
+            ++topk_tau_scored;
         }
         const double recall = recalls[i];
         if (!std::isnan(recall)) {
@@ -87,11 +118,10 @@ KnnScores evaluate_knn(const core::TrajectoryDb &compressed, const query::KnnWor
 
     KnnScores scores;
     scores.query_count = workload.entries.size();
-    scores.tau_query_count = tau_scored;
-    if (tau_scored != 0)
-        scores.kendalls_tau = tau_sum / static_cast<double>(tau_scored);
     if (recall_scored != 0)
         scores.recall_at_k = recall_sum / static_cast<double>(recall_scored);
+    if (topk_tau_scored != 0)
+        scores.kendalls_tau_topk = topk_tau_sum / static_cast<double>(topk_tau_scored);
     return scores;
 }
 

engine/src/pipeline/config.cpp

@@ -45,9 +45,10 @@ void parse_split_config(Config &config, const json &split) {
 }
 
 void parse_knn_config(Config &config, const json &knn) {
-    validate_keys(knn, {"k", "query_count"}, "knn");
+    validate_keys(knn, {"k", "query_count", "p"}, "knn");
     config.knn.k = knn.value("k", config.knn.k);
     config.knn.query_count = knn.value("query_count", config.knn.query_count);
+    config.knn.p = knn.value("p", config.knn.p);
 }
 
 void parse_range_config(Config &config, const json &range) {

engine/src/pipeline/pipeline.cpp

@@ -45,9 +45,9 @@ std::size_t total_budget(const core::TrajectoryDb &trajectories, double share) {
 }
 
 void print_scores(const evaluator::Scoretable &scores) {
-    std::cout << "  knn tau on shared neighbors: " << scores.knn.kendalls_tau << " ("
-              << scores.knn.tau_query_count << '/' << scores.knn.query_count << " queries)"
-              << ", recall@k: " << scores.knn.recall_at_k << '\n';
+    std::cout << "  knn top-k tau: " << scores.knn.kendalls_tau_topk
+              << ", recall@k: " << scores.knn.recall_at_k << " (" << scores.knn.query_count
+              << " queries)\n";
     std::cout << "  where-at: within radius=" << scores.where_at.within_radius
               << ", average hit=" << scores.where_at.average_hit << '\n';
     std::cout << "  range recall: " << scores.range_recall << '\n';

engine/src/query/knn.cpp

@@ -275,6 +275,9 @@ KnnWorkload generate_knn_queries(const core::TrajectoryDb &original,
     if (config.k == 0) {
         throw std::invalid_argument("knn: k = 0, no neighbors");
     }
+    if (config.p < 0.0 || config.p > 1.0) {
+        throw std::invalid_argument("knn: p must be in [0, 1]");
+    }
 
     std::vector<std::size_t> picked(query_pool.size());
     std::ranges::iota(picked.begin(), picked.end(), 0);
@@ -284,7 +287,7 @@ KnnWorkload generate_knn_queries(const core::TrajectoryDb &original,
         picked.resize(config.query_count);
     }
 
-    KnnWorkload results{core::subset(query_pool, picked), {}, config.k};
+    KnnWorkload results{core::subset(query_pool, picked), {}, config.k, config.p};
     results.entries.resize(results.queries.size());
 
     core::parallel_for(results.queries.size(), [&](std::size_t i) {

engine/tests/config_test.cpp

@@ -41,7 +41,7 @@ TEST(Config, ReadsTopLevelAndNestedKeys) {
         "split": {"type": "Time", "query_percentage": 0.3},
         "budget_shares": [0.5],
         "algorithms": ["SED"],
-        "knn": {"k": 3, "query_count": 0},
+        "knn": {"k": 3, "query_count": 0, "p": 0.25},
         "range": {"query_count": 10, "box_size_m": 500, "window_seconds": 3600},
         "where_at": {"probes_per_trajectory": 16, "max_gap_seconds": 30},
         "where_at_scoring": {"hit_radius_m": 50, "max_tolerance_m": 300}
@@ -53,6 +53,7 @@ TEST(Config, ReadsTopLevelAndNestedKeys) {
     EXPECT_EQ(config.algorithms, std::vector{selection::AlgorithmType::SED});
     EXPECT_EQ(config.knn.k, 3);
     EXPECT_EQ(config.knn.query_count, 0);
+    EXPECT_DOUBLE_EQ(config.knn.p, 0.25);
     EXPECT_EQ(config.range.query_count, 10);
     EXPECT_DOUBLE_EQ(config.range.box_size_m, 500);
     EXPECT_EQ(config.range.window_seconds, 3600);

engine/tests/evaluator_test.cpp

@@ -29,7 +29,7 @@ TEST(Evaluator, CollectsBothWhereAtScoresWithoutUnconfiguredQueries) {
     const auto scores = evaluator.evaluate();
     EXPECT_DOUBLE_EQ(scores.where_at.within_radius, 1);
     EXPECT_DOUBLE_EQ(scores.where_at.average_hit, 1);
-    EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau));
+    EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau_topk));
     EXPECT_TRUE(std::isnan(scores.knn.recall_at_k));
     EXPECT_TRUE(std::isnan(scores.range_recall));
 }
@@ -59,7 +59,7 @@ TEST(Evaluator, NoWorkloadsReturnsUndefinedScores) {
     const auto db = original();
     evaluator::Evaluator evaluator(db, db);
     const auto scores = evaluator.evaluate();
-    EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau));
+    EXPECT_TRUE(std::isnan(scores.knn.kendalls_tau_topk));
     EXPECT_TRUE(std::isnan(scores.knn.recall_at_k));
     EXPECT_TRUE(std::isnan(scores.range_recall));
     EXPECT_TRUE(std::isnan(scores.where_at.within_radius));
@@ -98,33 +98,41 @@ TEST(Evaluator, CollectsKnnAndWhereAtScoresTogether) {
     evaluator::Evaluator evaluator(db, db, {.knn = &knn, .where_at = where_at.queries});
 
     const auto scores = evaluator.evaluate();
-    EXPECT_DOUBLE_EQ(scores.knn.kendalls_tau, 1);
     EXPECT_DOUBLE_EQ(scores.knn.recall_at_k, 1);
+    EXPECT_DOUBLE_EQ(scores.knn.kendalls_tau_topk, 1);
     EXPECT_EQ(scores.knn.query_count, 2u);
-    EXPECT_EQ(scores.knn.tau_query_count, 2u);
     EXPECT_DOUBLE_EQ(scores.where_at.within_radius, 1);
     EXPECT_DOUBLE_EQ(scores.where_at.average_hit, 1);
-    EXPECT_DOUBLE_EQ(evaluator.get_knn_scores().kendalls_tau, scores.knn.kendalls_tau);
+    EXPECT_DOUBLE_EQ(evaluator.get_knn_scores().kendalls_tau_topk, scores.knn.kendalls_tau_topk);
     EXPECT_DOUBLE_EQ(evaluator.get_knn_scores().recall_at_k, scores.knn.recall_at_k);
 }
 
-TEST(KnnScores, ReportsCoverageWhenOnlySomeQueriesShareEnoughNeighbors) {
+TEST(KnnScores, AveragesOverQueriesWithPartialOverlap) {
     const core::TrajectoryDb db({0, 2, 4, 6, 8}, {10, 20, 30, 40}, {0, 10, 0, 10, 0, 10, 0, 10},
                                 std::vector<double>(8, 39.95),
                                 {116.35, 116.35, 116.36, 116.36, 116.37, 116.37, 116.38, 116.38});
     const query::KnnWorkload workload{db, {{0, {0, 1}}, {0, {0, 2}}, {0, {2, 3}}}, 2};
     const auto scores = evaluator::evaluate_knn(db, workload);
     EXPECT_EQ(scores.query_count, 3u);
-    EXPECT_EQ(scores.tau_query_count, 1u);
-    EXPECT_DOUBLE_EQ(scores.kendalls_tau, 1.0);
+    // the compressed answer is {0, 1}: identical, one swap-in (4/5 of max), disjoint
+    EXPECT_DOUBLE_EQ(scores.kendalls_tau_topk, (1.0 + 0.8 + 0.0) / 3);
     EXPECT_DOUBLE_EQ(scores.recall_at_k, 0.5);
 }
 
+TEST(KnnScores, TopkTauScoresPairsOutsideTheOverlap) {
+    // (7,3) is swapped, and 9 and 5 each only appear in one list: 2 of a possible 9 + 0.5 * 6
+    EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({7, 3, 9}, {3, 7, 5}, 0.5), 1.0 - 2.0 / 12);
+    EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({1, 2, 3}, {1, 2, 3}, 0.5), 1.0);
+    EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({1, 2}, {3, 4}, 0.5), 0.0);
+    // the simplified list holds neither 2 nor 3, so (2,3) costs p
+    EXPECT_DOUBLE_EQ(evaluator::knn::kendalls_tau_topk({1, 2, 3}, {1}, 1.0), 1.0 - 1.0 / 6);
+    EXPECT_TRUE(std::isnan(evaluator::knn::kendalls_tau_topk({}, {}, 0.5)));
+}
+
 TEST(KnnScores, EmptyWorkloadHasZeroCoverageAndUndefinedScores) {
     const auto db = original();
     const auto scores = evaluator::evaluate_knn(db, {db, {}, 2});
     EXPECT_EQ(scores.query_count, 0u);
-    EXPECT_EQ(scores.tau_query_count, 0u);
-    EXPECT_TRUE(std::isnan(scores.kendalls_tau));
+    EXPECT_TRUE(std::isnan(scores.kendalls_tau_topk));
     EXPECT_TRUE(std::isnan(scores.recall_at_k));
 }