diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b280268..4942e6e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,14 +118,16 @@ Serialization compatibility tests use snapshots from a pinned revision of [`apac The `cargo x prepare-testdata` command downloads the TCK archive and synchronizes its snapshots into: -- `datasketches/tests/serde_tests/java_generated_files` - `datasketches/tests/serde_tests/cpp_generated_files` +- `datasketches/tests/serde_tests/go_generated_files` +- `datasketches/tests/serde_tests/java_generated_files` You can synchronize them separately: ```shell -cargo x prepare-testdata java cargo x prepare-testdata cpp +cargo x prepare-testdata go +cargo x prepare-testdata java ``` If no language is specified, all languages are prepared. These directories are not stored in Git. Run the command before the first test run and again whenever the pinned TCK revision changes. It requires network access and replaces the selected generated directories. diff --git a/datasketches/tests/serde_tests.rs b/datasketches/tests/serde_tests.rs index 7348b0e..5f31c2d 100644 --- a/datasketches/tests/serde_tests.rs +++ b/datasketches/tests/serde_tests.rs @@ -33,7 +33,7 @@ pub fn serialization_test_data(sub_dir: &str, name: &str) -> PathBuf { run the following commands from the project root to prepare the test data files if they are missing: - $ cargo x prepare-testdata --all + $ cargo x prepare-testdata "#, path.display(), ); diff --git a/datasketches/tests/serde_tests/.gitignore b/datasketches/tests/serde_tests/.gitignore index 84c031c..d642f7a 100644 --- a/datasketches/tests/serde_tests/.gitignore +++ b/datasketches/tests/serde_tests/.gitignore @@ -1,2 +1,3 @@ cpp_generated_files +go_generated_files java_generated_files diff --git a/datasketches/tests/serde_tests/bloom.rs b/datasketches/tests/serde_tests/bloom.rs index 5875b1f..6e406ec 100644 --- a/datasketches/tests/serde_tests/bloom.rs +++ b/datasketches/tests/serde_tests/bloom.rs @@ -125,97 +125,53 @@ fn test_bloom_filter_file(path: PathBuf, expected_num_items: u64, expected_num_h } #[test] -fn test_java_bloom_n0_h3() { - let path = serialization_test_data("java_generated_files", "bf_n0_h3_java.sk"); - test_bloom_filter_file(path, 0, 3); -} - -#[test] -fn test_java_bloom_n0_h5() { - let path = serialization_test_data("java_generated_files", "bf_n0_h5_java.sk"); - test_bloom_filter_file(path, 0, 5); -} - -#[test] -fn test_java_bloom_n10000_h3() { - let path = serialization_test_data("java_generated_files", "bf_n10000_h3_java.sk"); - test_bloom_filter_file(path, 10000, 3); -} - -#[test] -fn test_java_bloom_n10000_h5() { - let path = serialization_test_data("java_generated_files", "bf_n10000_h5_java.sk"); - test_bloom_filter_file(path, 10000, 5); -} - -#[test] -fn test_java_bloom_n2000000_h3() { - let path = serialization_test_data("java_generated_files", "bf_n2000000_h3_java.sk"); - test_bloom_filter_file(path, 2000000, 3); -} - -#[test] -fn test_java_bloom_n2000000_h5() { - let path = serialization_test_data("java_generated_files", "bf_n2000000_h5_java.sk"); - test_bloom_filter_file(path, 2000000, 5); -} - -#[test] -fn test_java_bloom_n30000000_h3() { - let path = serialization_test_data("java_generated_files", "bf_n30000000_h3_java.sk"); - test_bloom_filter_file(path, 30000000, 3); -} - -#[test] -fn test_java_bloom_n30000000_h5() { - let path = serialization_test_data("java_generated_files", "bf_n30000000_h5_java.sk"); - test_bloom_filter_file(path, 30000000, 5); -} - -#[test] -fn test_cpp_bloom_n0_h3() { - let path = serialization_test_data("cpp_generated_files", "bf_n0_h3_cpp.sk"); - test_bloom_filter_file(path, 0, 3); -} - -#[test] -fn test_cpp_bloom_n0_h5() { - let path = serialization_test_data("cpp_generated_files", "bf_n0_h5_cpp.sk"); - test_bloom_filter_file(path, 0, 5); -} - -#[test] -fn test_cpp_bloom_n10000_h3() { - let path = serialization_test_data("cpp_generated_files", "bf_n10000_h3_cpp.sk"); - test_bloom_filter_file(path, 10000, 3); -} - -#[test] -fn test_cpp_bloom_n10000_h5() { - let path = serialization_test_data("cpp_generated_files", "bf_n10000_h5_cpp.sk"); - test_bloom_filter_file(path, 10000, 5); -} - -#[test] -fn test_cpp_bloom_n2000000_h3() { - let path = serialization_test_data("cpp_generated_files", "bf_n2000000_h3_cpp.sk"); - test_bloom_filter_file(path, 2000000, 3); -} - -#[test] -fn test_cpp_bloom_n2000000_h5() { - let path = serialization_test_data("cpp_generated_files", "bf_n2000000_h5_cpp.sk"); - test_bloom_filter_file(path, 2000000, 5); +fn test_java_compatibility() { + for (n, num_hashes) in [ + (0, 3), + (0, 5), + (10_000, 3), + (10_000, 5), + (2_000_000, 3), + (2_000_000, 5), + (30_000_000, 3), + (30_000_000, 5), + ] { + let filename = format!("bf_n{n}_h{num_hashes}_java.sk"); + let path = serialization_test_data("java_generated_files", &filename); + test_bloom_filter_file(path, n, num_hashes); + } } #[test] -fn test_cpp_bloom_n30000000_h3() { - let path = serialization_test_data("cpp_generated_files", "bf_n30000000_h3_cpp.sk"); - test_bloom_filter_file(path, 30000000, 3); +fn test_cpp_compatibility() { + for (n, num_hashes) in [ + (0, 3), + (0, 5), + (10_000, 3), + (10_000, 5), + (2_000_000, 3), + (2_000_000, 5), + (30_000_000, 3), + (30_000_000, 5), + ] { + let filename = format!("bf_n{n}_h{num_hashes}_cpp.sk"); + let path = serialization_test_data("cpp_generated_files", &filename); + test_bloom_filter_file(path, n, num_hashes); + } } #[test] -fn test_cpp_bloom_n30000000_h5() { - let path = serialization_test_data("cpp_generated_files", "bf_n30000000_h5_cpp.sk"); - test_bloom_filter_file(path, 30000000, 5); +fn test_go_compatibility() { + for (n, num_hashes) in [ + (0, 3), + (0, 5), + (10_000, 3), + (10_000, 5), + (2_000_000, 3), + (2_000_000, 5), + ] { + let filename = format!("bf_n{n}_h{num_hashes}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_bloom_filter_file(path, n, num_hashes); + } } diff --git a/datasketches/tests/serde_tests/cpc.rs b/datasketches/tests/serde_tests/cpc.rs index a99ca6c..cf27650 100644 --- a/datasketches/tests/serde_tests/cpc.rs +++ b/datasketches/tests/serde_tests/cpc.rs @@ -107,3 +107,15 @@ fn test_cpp_compatibility() { test_sketch_replay(&path, sketch, 1..=n); } } + +#[test] +fn test_go_compatibility() { + let test_cases = [0, 100, 200, 2000, 20000]; + + for n in test_cases { + let filename = format!("cpc_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + let sketch = test_sketch_file(&path, n); + test_sketch_replay(&path, sketch, 0..n); + } +} diff --git a/datasketches/tests/serde_tests/frequencies.rs b/datasketches/tests/serde_tests/frequencies.rs index 07051ba..31a80f8 100644 --- a/datasketches/tests/serde_tests/frequencies.rs +++ b/datasketches/tests/serde_tests/frequencies.rs @@ -264,3 +264,82 @@ fn test_cpp_frequent_strings_utf8() { assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6); assert_eq!(sketch.estimate(&"эюя".to_string()), 7); } + +#[test] +fn test_go_frequent_longs_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000]; + for n in test_cases { + let filename = format!("frequent_long_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + let bytes = fs::read(&path).unwrap(); + let sketch = FrequentItemsSketch::::deserialize(&bytes).unwrap(); + assert_eq!(sketch.is_empty(), n == 0); + if n > 10 { + assert!(sketch.maximum_error() > 0); + } else { + assert_eq!(sketch.maximum_error(), 0); + } + assert_eq!(sketch.total_weight(), n); + } +} + +#[test] +fn test_go_frequent_strings_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000]; + for n in test_cases { + let filename = format!("frequent_string_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + let bytes = fs::read(&path).unwrap(); + let sketch = FrequentItemsSketch::::deserialize(&bytes).unwrap(); + assert_eq!(sketch.is_empty(), n == 0); + if n > 10 { + assert!(sketch.maximum_error() > 0); + } else { + assert_eq!(sketch.maximum_error(), 0); + } + assert_eq!(sketch.total_weight(), n); + } +} + +#[test] +fn test_go_frequent_strings_ascii() { + let path = serialization_test_data("go_generated_files", "frequent_string_ascii_go.sk"); + let bytes = fs::read(&path).unwrap(); + let sketch = FrequentItemsSketch::::deserialize(&bytes).unwrap(); + assert!(!sketch.is_empty()); + assert_eq!(sketch.maximum_error(), 0); + assert_eq!(sketch.total_weight(), 10); + assert_eq!( + sketch.estimate(&"aaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()), + 1 + ); + assert_eq!( + sketch.estimate(&"bbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()), + 2 + ); + assert_eq!( + sketch.estimate(&"ccccccccccccccccccccccccccccc".to_string()), + 3 + ); + assert_eq!( + sketch.estimate(&"ddddddddddddddddddddddddddddd".to_string()), + 4 + ); +} + +#[test] +fn test_go_frequent_strings_utf8() { + let path = serialization_test_data("go_generated_files", "frequent_string_utf8_go.sk"); + let bytes = fs::read(&path).unwrap(); + let sketch = FrequentItemsSketch::::deserialize(&bytes).unwrap(); + assert!(!sketch.is_empty()); + assert_eq!(sketch.maximum_error(), 0); + assert_eq!(sketch.total_weight(), 28); + assert_eq!(sketch.estimate(&"абвгд".to_string()), 1); + assert_eq!(sketch.estimate(&"еёжзи".to_string()), 2); + assert_eq!(sketch.estimate(&"йклмн".to_string()), 3); + assert_eq!(sketch.estimate(&"опрст".to_string()), 4); + assert_eq!(sketch.estimate(&"уфхцч".to_string()), 5); + assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6); + assert_eq!(sketch.estimate(&"эюя".to_string()), 7); +} diff --git a/datasketches/tests/serde_tests/hll.rs b/datasketches/tests/serde_tests/hll.rs index fb1e4fb..3447c78 100644 --- a/datasketches/tests/serde_tests/hll.rs +++ b/datasketches/tests/serde_tests/hll.rs @@ -164,6 +164,7 @@ fn test_serialized_bytes_match_reference_files_for_coupon_modes() { for (dir, suffix) in [ ("java_generated_files", "java"), ("cpp_generated_files", "cpp"), + ("go_generated_files", "go"), ] { let filename = format!("{type_name}_n{n}_{suffix}.sk"); let path = serialization_test_data(dir, &filename); @@ -251,6 +252,39 @@ fn test_cpp_hll8_compatibility() { } } +#[test] +fn test_go_hll4_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000]; + + for n in test_cases { + let filename = format!("hll4_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, 12); + } +} + +#[test] +fn test_go_hll6_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000]; + + for n in test_cases { + let filename = format!("hll6_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, 12); + } +} + +#[test] +fn test_go_hll8_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000]; + + for n in test_cases { + let filename = format!("hll8_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, 12); + } +} + #[test] fn test_estimate_accuracy() { // This test verifies and prints actual estimates to show accuracy diff --git a/datasketches/tests/serde_tests/tdigest.rs b/datasketches/tests/serde_tests/tdigest.rs index a6251ab..7c860d7 100644 --- a/datasketches/tests/serde_tests/tdigest.rs +++ b/datasketches/tests/serde_tests/tdigest.rs @@ -131,6 +131,21 @@ fn test_deserialize_from_java_snapshots() { } } +#[test] +fn test_deserialize_from_go_snapshots() { + let ns = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000]; + for n in ns { + let filename = format!("tdigest_double_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, false, false); + } + for n in ns { + let filename = format!("tdigest_double_buf_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, true, false); + } +} + #[test] fn test_empty() { let mut td = TDigestMut::new(100); diff --git a/datasketches/tests/serde_tests/theta.rs b/datasketches/tests/serde_tests/theta.rs index b74cd44..1f0f09b 100644 --- a/datasketches/tests/serde_tests/theta.rs +++ b/datasketches/tests/serde_tests/theta.rs @@ -48,7 +48,7 @@ fn test_sketch_file(path: PathBuf, expected_cardinality: usize, use_compressed_r ) }); - // Theta snapshots from Java/C++ are not required to match byte-for-byte output + // Theta snapshots from other implementations are not required to match byte-for-byte output // from this implementation. Verify our own serialization is stable instead. let serialized_bytes2 = if use_compressed_round_trip { sketch2.serialize_compressed() @@ -116,6 +116,28 @@ fn test_cpp_compatibility() { test_sketch_file(path, 0, false); } +#[test] +fn test_go_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000]; + + for n in test_cases { + let filename = format!("theta_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, false); + } + + let compressed_test_cases = [10, 100, 1000, 10_000, 100_000, 1_000_000]; + + for n in compressed_test_cases { + let filename = format!("theta_compressed_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n, true); + } + + let path = serialization_test_data("go_generated_files", "theta_non_empty_no_entries_go.sk"); + test_sketch_file(path, 0, false); +} + #[test] fn malformed_input_is_rejected() { let mut sketch = ThetaSketchBuilder::default().lg_k(5).build(); diff --git a/datasketches/tests/serde_tests/tuple.rs b/datasketches/tests/serde_tests/tuple.rs index 03b0d7c..3dadf98 100644 --- a/datasketches/tests/serde_tests/tuple.rs +++ b/datasketches/tests/serde_tests/tuple.rs @@ -44,7 +44,7 @@ fn test_sketch_file(path: PathBuf, expected_cardinality: usize) { let estimate1 = sketch1.estimate(); assert_that!(estimate1, near(expected, expected * 0.03)); - // Snapshots from Java/C++ are not required to match byte-for-byte output from this + // Snapshots from other implementations are not required to match byte-for-byte output from this // implementation. Verify our own serialization is stable across a round-trip instead. let serialized_bytes = sketch1.serialize(); let sketch2 = CompactTupleSketch::::deserialize(&serialized_bytes).unwrap_or_else(|err| { @@ -94,6 +94,17 @@ fn test_cpp_compatibility() { } } +#[test] +fn test_go_compatibility() { + let test_cases = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000]; + + for n in test_cases { + let filename = format!("tuple_int_n{n}_go.sk"); + let path = serialization_test_data("go_generated_files", &filename); + test_sketch_file(path, n); + } +} + #[test] fn round_trip_preserves_summaries() { let mut sketch = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()).build(); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index a251ff6..50768fa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -283,7 +283,7 @@ fn make_taplo_cmd(fix: bool) -> StdCommand { struct CommandPrepareTestData { #[arg( value_name = "LANG", - value_parser = ["java", "cpp", "c++"], + value_parser = ["java", "cpp", "c++", "go", "golang"], help = "Languages to prepare (all by default)." )] langs: Vec, @@ -298,7 +298,7 @@ impl CommandPrepareTestData { } fn prepare(self) -> Result<()> { - const REVISION: &str = "0016a517cc87e13339298550afe8e6a7e961bf46"; + const REVISION: &str = "d363b12d293b395d90abb42677f9ea63178dbc0d"; let serde_tests = Path::new(env!("CARGO_WORKSPACE_DIR")).join("datasketches/tests/serde_tests"); let archive_url = @@ -372,7 +372,7 @@ impl CommandPrepareTestData { fn languages(&self) -> Vec<&'static str> { if self.langs.is_empty() { - return vec!["cpp", "java"]; + return vec!["cpp", "go", "java"]; } let mut languages = vec![]; @@ -380,6 +380,7 @@ impl CommandPrepareTestData { let language = match language.as_str() { "java" => "java", "cpp" | "c++" => "cpp", + "go" | "golang" => "go", _ => unreachable!("language is validated by clap"), }; if !languages.contains(&language) {