From 5c05f659172bcd2e739fc581a567afa33949c26e Mon Sep 17 00:00:00 2001 From: 17krishna8 <17krishna8@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:20:22 +0000 Subject: [PATCH 1/3] Clamp out-of-range alpha-scale radius in validate_config The a_scale_radius field is a public astcenc_config tuning field, and is also set directly by the CLI '-a ' option via atoi() with no bounds check, so it can hold any unsigned value. init_compute_averages() sizes the averaging pre-pass work memory as a product of padded block extents that grow linearly with the radius: work_memory_size = 2 * (32 + 2r + 1)^2 * (16 + 2r + 1) This size_t multiplication overflows for large radii, and even where it does not overflow the allocation grows without bound. A config with a large radius (e.g. the value 0xFFFFFFFF resulting from atoi() of a negative CLI argument, or a radius of 1000000) therefore causes astcenc_compress_image() to fail with an uncaught std::bad_alloc, which terminates the process rather than returning an error code. Clamp the radius in validate_config() to the largest supported value, following the existing pattern for the other tunable config fields. Worst-case work memory after the clamp is around 100 MB for a 3D image, and any practical alpha-scale use case uses single-digit radii. --- Source/astcenc_entry.cpp | 7 +++++++ Source/astcenc_internal.h | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/Source/astcenc_entry.cpp b/Source/astcenc_entry.cpp index 79c65a0c..b62fb6c1 100644 --- a/Source/astcenc_entry.cpp +++ b/Source/astcenc_entry.cpp @@ -464,6 +464,13 @@ static astcenc_error validate_config( config.rgbm_m_scale = astc::max(config.rgbm_m_scale, 1.0f); + // The alpha-scale kernel radius is a public tuning field, so may be set to + // any value by an application. The averaging pre-pass sizes its work memory + // as a product of padded block extents that grow with the radius, so clamp + // the radius to the supported maximum to keep that multiplication bounded + // for any image size + config.a_scale_radius = astc::min(config.a_scale_radius, TUNE_MAX_A_SCALE_RADIUS); + config.tune_partition_count_limit = astc::clamp(config.tune_partition_count_limit, 1u, 4u); config.tune_2partition_index_limit = astc::clamp(config.tune_2partition_index_limit, 1u, BLOCK_MAX_PARTITIONINGS); config.tune_3partition_index_limit = astc::clamp(config.tune_3partition_index_limit, 1u, BLOCK_MAX_PARTITIONINGS); diff --git a/Source/astcenc_internal.h b/Source/astcenc_internal.h index 46942171..0b6124c6 100644 --- a/Source/astcenc_internal.h +++ b/Source/astcenc_internal.h @@ -125,6 +125,14 @@ static constexpr float ERROR_CALC_DEFAULT { 1e30f }; */ static constexpr float TUNE_MIN_SEARCH_MODE0 { 0.85f }; +/** + * @brief The maximum alpha-scale kernel radius supported by alpha-weighted compression. + * + * The work memory needed by the averaging pre-pass grows multiplicatively with + * the radius, so out-of-range values are clamped to keep allocations bounded. + */ +static constexpr unsigned int TUNE_MAX_A_SCALE_RADIUS { 64 }; + /** * @brief The maximum number of candidate encodings tested for each encoding mode. * From 3c716944491b26a10de54a11ea4b753111fa3391 Mon Sep 17 00:00:00 2001 From: 17krishna8 <17krishna8@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:31:56 +0000 Subject: [PATCH 2/3] Add unit test for an out-of-range alpha-scale radius --- Source/UnitTest/test_encode.cpp | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Source/UnitTest/test_encode.cpp b/Source/UnitTest/test_encode.cpp index 069f6441..42559a1c 100644 --- a/Source/UnitTest/test_encode.cpp +++ b/Source/UnitTest/test_encode.cpp @@ -62,6 +62,42 @@ TEST(compress, overflow_in_z) astcenc_context_free(context); } +/** @brief Input validation tests for an out-of-range alpha-scale radius. */ +TEST(compress, ascale_radius_clamped) +{ + astcenc_error status; + astcenc_config config; + astcenc_context* context; + + static const astcenc_swizzle swizzle { + ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A + }; + + astcenc_config_init(ASTCENC_PRF_LDR, 4, 4, 1, ASTCENC_PRE_MEDIUM, 0, &config); + // Any unsigned value is a legal field value; large ones must be clamped + config.a_scale_radius = 0xFFFFFFFFu; + status = astcenc_context_alloc(&config, 1, &context, nullptr); + EXPECT_EQ(status, ASTCENC_SUCCESS); + + uint8_t data[16]; + uint8_t input[4 * 4 * 4]; + + astcenc_image image; + image.dim_x = 4; + image.dim_y = 4; + image.dim_z = 1; + image.data_type = ASTCENC_TYPE_U8; + uint8_t* slices = input; + image.data = reinterpret_cast(&slices); + + // The averaging pre-pass sizes work memory from the radius, which must + // stay bounded instead of overflowing or throwing std::bad_alloc + status = astcenc_compress_image(context, &image, &swizzle, data, sizeof(data), 0); + EXPECT_EQ(status, ASTCENC_SUCCESS); + + astcenc_context_free(context); +} + /** @brief Input overflow tests for xyz * 16. */ TEST(compress, overflow_in_16) { From 94c0f70942ccd8a7341b84a765baf9622eb1b241 Mon Sep 17 00:00:00 2001 From: Peter Harris Date: Wed, 21 Oct 2026 11:58:25 +0100 Subject: [PATCH 3/3] Cleanup comments and rename new config option as its not a tuning option --- Source/UnitTest/test_encode.cpp | 49 ++++++++++++++++----------------- Source/astcenc_entry.cpp | 8 ++---- Source/astcenc_internal.h | 11 ++------ 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/Source/UnitTest/test_encode.cpp b/Source/UnitTest/test_encode.cpp index 42559a1c..cc9be93c 100644 --- a/Source/UnitTest/test_encode.cpp +++ b/Source/UnitTest/test_encode.cpp @@ -62,8 +62,8 @@ TEST(compress, overflow_in_z) astcenc_context_free(context); } -/** @brief Input validation tests for an out-of-range alpha-scale radius. */ -TEST(compress, ascale_radius_clamped) +/** @brief Input overflow tests for xyz * 16. */ +TEST(compress, overflow_in_16) { astcenc_error status; astcenc_config config; @@ -74,32 +74,30 @@ TEST(compress, ascale_radius_clamped) }; astcenc_config_init(ASTCENC_PRF_LDR, 4, 4, 1, ASTCENC_PRE_MEDIUM, 0, &config); - // Any unsigned value is a legal field value; large ones must be clamped - config.a_scale_radius = 0xFFFFFFFFu; status = astcenc_context_alloc(&config, 1, &context, nullptr); EXPECT_EQ(status, ASTCENC_SUCCESS); - uint8_t data[16]; - uint8_t input[4 * 4 * 4]; + // Arrays are too short, but should never be touched + uint8_t data[1]; + uint8_t input[1]; astcenc_image image; - image.dim_x = 4; - image.dim_y = 4; - image.dim_z = 1; + // xyz (in blocks) always fits in 64-bit size_t, but xyz * 16 will overflow + image.dim_x = 0x80000000u; + image.dim_y = 0x80000000u; + image.dim_z = 0x00000010u; image.data_type = ASTCENC_TYPE_U8; uint8_t* slices = input; image.data = reinterpret_cast(&slices); - // The averaging pre-pass sizes work memory from the radius, which must - // stay bounded instead of overflowing or throwing std::bad_alloc - status = astcenc_compress_image(context, &image, &swizzle, data, sizeof(data), 0); - EXPECT_EQ(status, ASTCENC_SUCCESS); + status = astcenc_compress_image(context, &image, &swizzle, data, -1, 0); + EXPECT_EQ(status, ASTCENC_ERR_BAD_PARAM); astcenc_context_free(context); } -/** @brief Input overflow tests for xyz * 16. */ -TEST(compress, overflow_in_16) +/** @brief Input test for an out-of-range alpha-scale radius. */ +TEST(compress, ascale_radius_clamped) { astcenc_error status; astcenc_config config; @@ -110,29 +108,30 @@ TEST(compress, overflow_in_16) }; astcenc_config_init(ASTCENC_PRF_LDR, 4, 4, 1, ASTCENC_PRE_MEDIUM, 0, &config); + + // Set to largest possible value to check input validation + config.a_scale_radius = 0xFFFFFFFFu; status = astcenc_context_alloc(&config, 1, &context, nullptr); EXPECT_EQ(status, ASTCENC_SUCCESS); - // Arrays are too short, but should never be touched - uint8_t data[1]; - uint8_t input[1]; + uint8_t data[16]; + uint8_t input[4 * 4 * 4]; astcenc_image image; - // xyz (in blocks) always fits in 64-bit size_t, but xyz * 16 will overflow - image.dim_x = 0x80000000u; - image.dim_y = 0x80000000u; - image.dim_z = 0x00000010u; + image.dim_x = 4; + image.dim_y = 4; + image.dim_z = 1; image.data_type = ASTCENC_TYPE_U8; uint8_t* slices = input; image.data = reinterpret_cast(&slices); - status = astcenc_compress_image(context, &image, &swizzle, data, -1, 0); - EXPECT_EQ(status, ASTCENC_ERR_BAD_PARAM); + // Compression should succeed without bad_alloc or ASAN failure + status = astcenc_compress_image(context, &image, &swizzle, data, sizeof(data), 0); + EXPECT_EQ(status, ASTCENC_SUCCESS); astcenc_context_free(context); } - /** @brief Input data buffer overrun tests. */ TEST(compress, data_buffer_exceeded) { diff --git a/Source/astcenc_entry.cpp b/Source/astcenc_entry.cpp index b62fb6c1..3e7dddf1 100644 --- a/Source/astcenc_entry.cpp +++ b/Source/astcenc_entry.cpp @@ -464,12 +464,8 @@ static astcenc_error validate_config( config.rgbm_m_scale = astc::max(config.rgbm_m_scale, 1.0f); - // The alpha-scale kernel radius is a public tuning field, so may be set to - // any value by an application. The averaging pre-pass sizes its work memory - // as a product of padded block extents that grow with the radius, so clamp - // the radius to the supported maximum to keep that multiplication bounded - // for any image size - config.a_scale_radius = astc::min(config.a_scale_radius, TUNE_MAX_A_SCALE_RADIUS); + // Clamp alpha-scale kernel radius to put an upper bound on memory use + config.a_scale_radius = astc::min(config.a_scale_radius, ALPHASCALE_MAX_RADIUS); config.tune_partition_count_limit = astc::clamp(config.tune_partition_count_limit, 1u, 4u); config.tune_2partition_index_limit = astc::clamp(config.tune_2partition_index_limit, 1u, BLOCK_MAX_PARTITIONINGS); diff --git a/Source/astcenc_internal.h b/Source/astcenc_internal.h index 0b6124c6..ea3c1a42 100644 --- a/Source/astcenc_internal.h +++ b/Source/astcenc_internal.h @@ -120,19 +120,14 @@ static constexpr unsigned int WEIGHTS_MAX_DECIMATION_MODES { 87 }; /** @brief The high default error used to initialize error trackers. */ static constexpr float ERROR_CALC_DEFAULT { 1e30f }; +/** @brief The maximum alpha-scale kernel radius. */ +static constexpr unsigned int ALPHASCALE_MAX_RADIUS { 64 }; + /** * @brief The minimum tuning setting threshold for the one partition fast path. */ static constexpr float TUNE_MIN_SEARCH_MODE0 { 0.85f }; -/** - * @brief The maximum alpha-scale kernel radius supported by alpha-weighted compression. - * - * The work memory needed by the averaging pre-pass grows multiplicatively with - * the radius, so out-of-range values are clamped to keep allocations bounded. - */ -static constexpr unsigned int TUNE_MAX_A_SCALE_RADIUS { 64 }; - /** * @brief The maximum number of candidate encodings tested for each encoding mode. *