test: cover Go serialization snapshots

CONTRIBUTING.md

@@ -118,14 +118,16 @@ Serialization compatibility tests use snapshots from a pinned revision of [`apac
 
 The `cargo x prepare-testdata` command downloads the TCK archive and synchronizes its snapshots into:
 
-- `datasketches/tests/serde_tests/java_generated_files`
 - `datasketches/tests/serde_tests/cpp_generated_files`
+- `datasketches/tests/serde_tests/go_generated_files`
+- `datasketches/tests/serde_tests/java_generated_files`
 
 You can synchronize them separately:
 
 ```shell
-cargo x prepare-testdata java
 cargo x prepare-testdata cpp
+cargo x prepare-testdata go
+cargo x prepare-testdata java
 ```
 
 If no language is specified, all languages are prepared. These directories are not stored in Git. Run the command before the first test run and again whenever the pinned TCK revision changes. It requires network access and replaces the selected generated directories.

datasketches/tests/serde_tests.rs

@@ -33,7 +33,7 @@ pub fn serialization_test_data(sub_dir: &str, name: &str) -> PathBuf {
             run the following commands from the project root to prepare the test data files
             if they are missing:
 
-            $ cargo x prepare-testdata --all
+            $ cargo x prepare-testdata
         "#,
             path.display(),
         );

datasketches/tests/serde_tests/.gitignore

@@ -1,2 +1,3 @@
 cpp_generated_files
+go_generated_files
 java_generated_files

datasketches/tests/serde_tests/bloom.rs

@@ -125,97 +125,53 @@ fn test_bloom_filter_file(path: PathBuf, expected_num_items: u64, expected_num_h
 }
 
 #[test]
-fn test_java_bloom_n0_h3() {
-    let path = serialization_test_data("java_generated_files", "bf_n0_h3_java.sk");
-    test_bloom_filter_file(path, 0, 3);
-}
-
-#[test]
-fn test_java_bloom_n0_h5() {
-    let path = serialization_test_data("java_generated_files", "bf_n0_h5_java.sk");
-    test_bloom_filter_file(path, 0, 5);
-}
-
-#[test]
-fn test_java_bloom_n10000_h3() {
-    let path = serialization_test_data("java_generated_files", "bf_n10000_h3_java.sk");
-    test_bloom_filter_file(path, 10000, 3);
-}
-
-#[test]
-fn test_java_bloom_n10000_h5() {
-    let path = serialization_test_data("java_generated_files", "bf_n10000_h5_java.sk");
-    test_bloom_filter_file(path, 10000, 5);
-}
-
-#[test]
-fn test_java_bloom_n2000000_h3() {
-    let path = serialization_test_data("java_generated_files", "bf_n2000000_h3_java.sk");
-    test_bloom_filter_file(path, 2000000, 3);
-}
-
-#[test]
-fn test_java_bloom_n2000000_h5() {
-    let path = serialization_test_data("java_generated_files", "bf_n2000000_h5_java.sk");
-    test_bloom_filter_file(path, 2000000, 5);
-}
-
-#[test]
-fn test_java_bloom_n30000000_h3() {
-    let path = serialization_test_data("java_generated_files", "bf_n30000000_h3_java.sk");
-    test_bloom_filter_file(path, 30000000, 3);
-}
-
-#[test]
-fn test_java_bloom_n30000000_h5() {
-    let path = serialization_test_data("java_generated_files", "bf_n30000000_h5_java.sk");
-    test_bloom_filter_file(path, 30000000, 5);
-}
-
-#[test]
-fn test_cpp_bloom_n0_h3() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n0_h3_cpp.sk");
-    test_bloom_filter_file(path, 0, 3);
-}
-
-#[test]
-fn test_cpp_bloom_n0_h5() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n0_h5_cpp.sk");
-    test_bloom_filter_file(path, 0, 5);
-}
-
-#[test]
-fn test_cpp_bloom_n10000_h3() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n10000_h3_cpp.sk");
-    test_bloom_filter_file(path, 10000, 3);
-}
-
-#[test]
-fn test_cpp_bloom_n10000_h5() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n10000_h5_cpp.sk");
-    test_bloom_filter_file(path, 10000, 5);
-}
-
-#[test]
-fn test_cpp_bloom_n2000000_h3() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n2000000_h3_cpp.sk");
-    test_bloom_filter_file(path, 2000000, 3);
-}
-
-#[test]
-fn test_cpp_bloom_n2000000_h5() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n2000000_h5_cpp.sk");
-    test_bloom_filter_file(path, 2000000, 5);
+fn test_java_compatibility() {
+    for (n, num_hashes) in [
+        (0, 3),
+        (0, 5),
+        (10_000, 3),
+        (10_000, 5),
+        (2_000_000, 3),
+        (2_000_000, 5),
+        (30_000_000, 3),
+        (30_000_000, 5),
+    ] {
+        let filename = format!("bf_n{n}_h{num_hashes}_java.sk");
+        let path = serialization_test_data("java_generated_files", &filename);
+        test_bloom_filter_file(path, n, num_hashes);
+    }
 }
 
 #[test]
-fn test_cpp_bloom_n30000000_h3() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n30000000_h3_cpp.sk");
-    test_bloom_filter_file(path, 30000000, 3);
+fn test_cpp_compatibility() {
+    for (n, num_hashes) in [
+        (0, 3),
+        (0, 5),
+        (10_000, 3),
+        (10_000, 5),
+        (2_000_000, 3),
+        (2_000_000, 5),
+        (30_000_000, 3),
+        (30_000_000, 5),
+    ] {
+        let filename = format!("bf_n{n}_h{num_hashes}_cpp.sk");
+        let path = serialization_test_data("cpp_generated_files", &filename);
+        test_bloom_filter_file(path, n, num_hashes);
+    }
 }
 
 #[test]
-fn test_cpp_bloom_n30000000_h5() {
-    let path = serialization_test_data("cpp_generated_files", "bf_n30000000_h5_cpp.sk");
-    test_bloom_filter_file(path, 30000000, 5);
+fn test_go_compatibility() {
+    for (n, num_hashes) in [
+        (0, 3),
+        (0, 5),
+        (10_000, 3),
+        (10_000, 5),
+        (2_000_000, 3),
+        (2_000_000, 5),
+    ] {
+        let filename = format!("bf_n{n}_h{num_hashes}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_bloom_filter_file(path, n, num_hashes);
+    }
 }

datasketches/tests/serde_tests/cpc.rs

@@ -107,3 +107,15 @@ fn test_cpp_compatibility() {
         test_sketch_replay(&path, sketch, 1..=n);
     }
 }
+
+#[test]
+fn test_go_compatibility() {
+    let test_cases = [0, 100, 200, 2000, 20000];
+
+    for n in test_cases {
+        let filename = format!("cpc_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        let sketch = test_sketch_file(&path, n);
+        test_sketch_replay(&path, sketch, 0..n);
+    }
+}

datasketches/tests/serde_tests/frequencies.rs

@@ -264,3 +264,82 @@ fn test_cpp_frequent_strings_utf8() {
     assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6);
     assert_eq!(sketch.estimate(&"эюя".to_string()), 7);
 }
+
+#[test]
+fn test_go_frequent_longs_compatibility() {
+    let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+    for n in test_cases {
+        let filename = format!("frequent_long_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        let bytes = fs::read(&path).unwrap();
+        let sketch = FrequentItemsSketch::<i64>::deserialize(&bytes).unwrap();
+        assert_eq!(sketch.is_empty(), n == 0);
+        if n > 10 {
+            assert!(sketch.maximum_error() > 0);
+        } else {
+            assert_eq!(sketch.maximum_error(), 0);
+        }
+        assert_eq!(sketch.total_weight(), n);
+    }
+}
+
+#[test]
+fn test_go_frequent_strings_compatibility() {
+    let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+    for n in test_cases {
+        let filename = format!("frequent_string_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        let bytes = fs::read(&path).unwrap();
+        let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+        assert_eq!(sketch.is_empty(), n == 0);
+        if n > 10 {
+            assert!(sketch.maximum_error() > 0);
+        } else {
+            assert_eq!(sketch.maximum_error(), 0);
+        }
+        assert_eq!(sketch.total_weight(), n);
+    }
+}
+
+#[test]
+fn test_go_frequent_strings_ascii() {
+    let path = serialization_test_data("go_generated_files", "frequent_string_ascii_go.sk");
+    let bytes = fs::read(&path).unwrap();
+    let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+    assert!(!sketch.is_empty());
+    assert_eq!(sketch.maximum_error(), 0);
+    assert_eq!(sketch.total_weight(), 10);
+    assert_eq!(
+        sketch.estimate(&"aaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()),
+        1
+    );
+    assert_eq!(
+        sketch.estimate(&"bbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()),
+        2
+    );
+    assert_eq!(
+        sketch.estimate(&"ccccccccccccccccccccccccccccc".to_string()),
+        3
+    );
+    assert_eq!(
+        sketch.estimate(&"ddddddddddddddddddddddddddddd".to_string()),
+        4
+    );
+}
+
+#[test]
+fn test_go_frequent_strings_utf8() {
+    let path = serialization_test_data("go_generated_files", "frequent_string_utf8_go.sk");
+    let bytes = fs::read(&path).unwrap();
+    let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+    assert!(!sketch.is_empty());
+    assert_eq!(sketch.maximum_error(), 0);
+    assert_eq!(sketch.total_weight(), 28);
+    assert_eq!(sketch.estimate(&"абвгд".to_string()), 1);
+    assert_eq!(sketch.estimate(&"еёжзи".to_string()), 2);
+    assert_eq!(sketch.estimate(&"йклмн".to_string()), 3);
+    assert_eq!(sketch.estimate(&"опрст".to_string()), 4);
+    assert_eq!(sketch.estimate(&"уфхцч".to_string()), 5);
+    assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6);
+    assert_eq!(sketch.estimate(&"эюя".to_string()), 7);
+}

datasketches/tests/serde_tests/hll.rs

@@ -164,6 +164,7 @@ fn test_serialized_bytes_match_reference_files_for_coupon_modes() {
             for (dir, suffix) in [
                 ("java_generated_files", "java"),
                 ("cpp_generated_files", "cpp"),
+                ("go_generated_files", "go"),
             ] {
                 let filename = format!("{type_name}_n{n}_{suffix}.sk");
                 let path = serialization_test_data(dir, &filename);
@@ -251,6 +252,39 @@ fn test_cpp_hll8_compatibility() {
     }
 }
 
+#[test]
+fn test_go_hll4_compatibility() {
+    let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+
+    for n in test_cases {
+        let filename = format!("hll4_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, 12);
+    }
+}
+
+#[test]
+fn test_go_hll6_compatibility() {
+    let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+
+    for n in test_cases {
+        let filename = format!("hll6_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, 12);
+    }
+}
+
+#[test]
+fn test_go_hll8_compatibility() {
+    let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+
+    for n in test_cases {
+        let filename = format!("hll8_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, 12);
+    }
+}
+
 #[test]
 fn test_estimate_accuracy() {
     // This test verifies and prints actual estimates to show accuracy

datasketches/tests/serde_tests/tdigest.rs

@@ -131,6 +131,21 @@ fn test_deserialize_from_java_snapshots() {
     }
 }
 
+#[test]
+fn test_deserialize_from_go_snapshots() {
+    let ns = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000];
+    for n in ns {
+        let filename = format!("tdigest_double_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, false, false);
+    }
+    for n in ns {
+        let filename = format!("tdigest_double_buf_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, true, false);
+    }
+}
+
 #[test]
 fn test_empty() {
     let mut td = TDigestMut::new(100);

datasketches/tests/serde_tests/theta.rs

@@ -48,7 +48,7 @@ fn test_sketch_file(path: PathBuf, expected_cardinality: usize, use_compressed_r
         )
     });
 
-    // Theta snapshots from Java/C++ are not required to match byte-for-byte output
+    // Theta snapshots from other implementations are not required to match byte-for-byte output
     // from this implementation. Verify our own serialization is stable instead.
     let serialized_bytes2 = if use_compressed_round_trip {
         sketch2.serialize_compressed()
@@ -116,6 +116,28 @@ fn test_cpp_compatibility() {
     test_sketch_file(path, 0, false);
 }
 
+#[test]
+fn test_go_compatibility() {
+    let test_cases = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000];
+
+    for n in test_cases {
+        let filename = format!("theta_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, false);
+    }
+
+    let compressed_test_cases = [10, 100, 1000, 10_000, 100_000, 1_000_000];
+
+    for n in compressed_test_cases {
+        let filename = format!("theta_compressed_n{n}_go.sk");
+        let path = serialization_test_data("go_generated_files", &filename);
+        test_sketch_file(path, n, true);
+    }
+
+    let path = serialization_test_data("go_generated_files", "theta_non_empty_no_entries_go.sk");
+    test_sketch_file(path, 0, false);
+}
+
 #[test]
 fn malformed_input_is_rejected() {
     let mut sketch = ThetaSketchBuilder::default().lg_k(5).build();