From 76dc3f582448fabc8aba37e8286467c10b96b8d2 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 26 Oct 2026 12:27:10 -0400 Subject: [PATCH] Document that RunOptimize recovers space after modifying run containers --- BitSliceIndexing/bsi.go | 4 +++- README.md | 9 +++++++++ roaring.go | 11 ++++++++++- roaring64/bsi64.go | 4 +++- roaring64/roaring64.go | 11 ++++++++++- roaring64/roaring64_test.go | 14 ++++++++++++++ roaring_test.go | 15 +++++++++++++++ 7 files changed, 64 insertions(+), 4 deletions(-) diff --git a/BitSliceIndexing/bsi.go b/BitSliceIndexing/bsi.go index a58bcae5..158e1a85 100644 --- a/BitSliceIndexing/bsi.go +++ b/BitSliceIndexing/bsi.go @@ -54,7 +54,9 @@ func NewDefaultBSI() *BSI { return NewBSI(int64(0), int64(0)) } -// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap +// RunOptimize converts the containers of the underlying bitmaps to their most compact +// encoding. Calling it again after modifying the BSI can recover space: see +// Bitmap.RunOptimize. func (b *BSI) RunOptimize() { b.eBM.RunOptimize() for i := 0; i < len(b.bA); i++ { diff --git a/README.md b/README.md index 7957a463..20ae073b 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,15 @@ That is, given a fixed overhead for the universe size (x), Roaring bitmaps never use more than 2 bytes per integer. You can call ``BoundSerializedSizeInBytes`` for a more precise estimate. +Call ``RunOptimize`` to convert containers to their most compact encoding, +including run-length encoding where it helps. Some operations that modify a +bitmap, such as ``AddRange`` and ``RemoveRange``, do not always convert a run +container back to an array or bitmap container when the runs stop being the +most compact encoding. In the worst case, the runs of one 65536-value chunk can +take about 128 KiB where a bitmap container takes 8 KiB. Calling ``RunOptimize`` +again after such modifications recovers that space, in memory and once +serialized. This applies to 64-bit bitmaps (``roaring64``) as well. + ### 64-bit Roaring By default, roaring is used to stored unsigned 32-bit integers. However, we also offer diff --git a/roaring.go b/roaring.go index 8f80e838..b47c5404 100644 --- a/roaring.go +++ b/roaring.go @@ -394,7 +394,16 @@ func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) { return } -// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap +// RunOptimize converts each container to its most compact encoding: run-length encoding +// where it is smaller, and an array or bitmap container where it is not. +// +// Some operations that modify a bitmap, such as AddRange and RemoveRange, do not always +// convert a run container back to an array or bitmap container when the runs stop being +// the most compact encoding. Such a container can then use more memory and serialized +// space than needed: in the worst case, the runs of one 65536-value chunk can take about +// 128 KiB where a bitmap container takes 8 KiB. Calling RunOptimize again after such +// modifications recovers that space. Consider calling it before serializing a bitmap that +// was modified since its last call. func (rb *Bitmap) RunOptimize() { rb.highlowcontainer.runOptimize() } diff --git a/roaring64/bsi64.go b/roaring64/bsi64.go index 4b66e75d..f37d5f92 100644 --- a/roaring64/bsi64.go +++ b/roaring64/bsi64.go @@ -51,7 +51,9 @@ func NewDefaultBSI() *BSI { return NewBSI(int64(0), int64(0)) } -// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap +// RunOptimize converts the containers of the underlying bitmaps to their most compact +// encoding. Calling it again after modifying the BSI can recover space: see +// Bitmap.RunOptimize. func (b *BSI) RunOptimize() { b.eBM.RunOptimize() for i := 0; i < len(b.bA); i++ { diff --git a/roaring64/roaring64.go b/roaring64/roaring64.go index 143209de..ef13b08f 100644 --- a/roaring64/roaring64.go +++ b/roaring64/roaring64.go @@ -187,7 +187,16 @@ func (rb *Bitmap) UnmarshalBinary(data []byte) error { return err } -// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap +// RunOptimize converts each container to its most compact encoding: run-length encoding +// where it is smaller, and an array or bitmap container where it is not. +// +// Some operations that modify a bitmap, such as AddRange and RemoveRange, do not always +// convert a run container back to an array or bitmap container when the runs stop being +// the most compact encoding. Such a container can then use more memory and serialized +// space than needed: in the worst case, the runs of one 65536-value chunk can take about +// 128 KiB where a bitmap container takes 8 KiB. Calling RunOptimize again after such +// modifications recovers that space. Consider calling it before serializing a bitmap that +// was modified since its last call. func (rb *Bitmap) RunOptimize() { rb.highlowcontainer.runOptimize() } diff --git a/roaring64/roaring64_test.go b/roaring64/roaring64_test.go index cb4db9db..106303ce 100644 --- a/roaring64/roaring64_test.go +++ b/roaring64/roaring64_test.go @@ -2141,3 +2141,17 @@ func TestRoaringArray64SortOrder(t *testing.T) { assert.True(t, bm.highlowcontainer.checkKeysSorted()) }) } + +func TestRunOptimizeRecoversSpaceAfterRemoveRange(t *testing.T) { + bm := New() + bm.AddRange(0, 65536) + for i := uint64(0); i < 65536; i += 2 { + bm.RemoveRange(i, i+1) // each remaining value becomes a run of length one + } + expected := bm.Clone() + before := bm.GetSerializedSizeInBytes() + bm.RunOptimize() + assert.Less(t, bm.GetSerializedSizeInBytes(), before/10) + assert.False(t, bm.HasRunCompression()) + assert.True(t, expected.Equals(bm)) +} diff --git a/roaring_test.go b/roaring_test.go index 3d6ed1df..5ccbdd32 100644 --- a/roaring_test.go +++ b/roaring_test.go @@ -4211,3 +4211,18 @@ func TestValidateAcceptsRunContainersThatAreNotTheSmallestEncoding(t *testing.T) assert.NoError(t, deserialized.Validate()) assert.True(t, expected.Equals(deserialized)) } + +func TestRunOptimizeRecoversSpaceAfterRemoveRange(t *testing.T) { + bm := New() + bm.AddRange(0, 65536) + for i := uint64(0); i < 65536; i += 2 { + bm.RemoveRange(i, i+1) // each remaining value becomes a run of length one + } + expected := bm.Clone() + before := bm.GetSerializedSizeInBytes() + bm.RunOptimize() + assert.Less(t, bm.GetSerializedSizeInBytes(), before/10) + assert.False(t, bm.HasRunCompression()) + assert.True(t, expected.Equals(bm)) + require.NoError(t, bm.Validate()) +}