From 06eb84c7373fa608838cda8c27635d44cd9851e0 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 11:49:12 +0000 Subject: [PATCH 01/24] Move allocconfig.h from ds/ to ds_core/ Pure file move. allocconfig.h contains only compile-time constants with dependencies only on bits.h and mitigations.h (both in ds_core/). Moving it enables sizeclassstatic.h (next step) to live in ds_core/ without pulling in ds/. Changes: - git mv ds/allocconfig.h -> ds_core/allocconfig.h - Add #include bits.h and mitigations.h to moved file - Update include paths in ds/ds.h, ds/mpmcstack.h - Add allocconfig.h to ds_core/ds_core.h umbrella --- src/snmalloc/ds/ds.h | 1 - src/snmalloc/ds/mpmcstack.h | 1 - src/snmalloc/{ds => ds_core}/allocconfig.h | 3 +++ src/snmalloc/ds_core/ds_core.h | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) rename src/snmalloc/{ds => ds_core}/allocconfig.h (99%) diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index e24b1f25b..47f097fc0 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -6,7 +6,6 @@ #include "../ds_aal/ds_aal.h" #include "../pal/pal.h" #include "aba.h" -#include "allocconfig.h" #include "combininglock.h" #include "entropy.h" #include "mpmcstack.h" diff --git a/src/snmalloc/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h index f1e965e53..57e587637 100644 --- a/src/snmalloc/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -2,7 +2,6 @@ #include "../ds_core/ds_core.h" #include "aba.h" -#include "allocconfig.h" namespace snmalloc { diff --git a/src/snmalloc/ds/allocconfig.h b/src/snmalloc/ds_core/allocconfig.h similarity index 99% rename from src/snmalloc/ds/allocconfig.h rename to src/snmalloc/ds_core/allocconfig.h index d58d863b8..5ee00af76 100644 --- a/src/snmalloc/ds/allocconfig.h +++ b/src/snmalloc/ds_core/allocconfig.h @@ -1,5 +1,8 @@ #pragma once +#include "bits.h" +#include "mitigations.h" + namespace snmalloc { // 0 intermediate bits results in power of 2 small allocs. 1 intermediate diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index cc395127b..fc9bd5be0 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -7,6 +7,7 @@ * snmalloc. */ +#include "allocconfig.h" #include "bits.h" #include "cheri.h" #include "concept.h" From fad2d2a42a68b7f94196870215db55ea9534e078 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 17:51:00 +0000 Subject: [PATCH 02/24] Extract sizeclassstatic.h into ds_core/, move sizeclasstable.h to ds/ Create ds_core/sizeclassstatic.h with compile-time sizeclass functions extracted from mem/sizeclasstable.h: - smallsizeclass_t (size_t typedef) - size_to_sizeclass_const() - NUM_SMALL_SIZECLASSES - sizeclass_to_size_const() - is_small_sizeclass() Move sizeclasstable.h from mem/ to ds/ (depends on PAL, not allocator). Remove redundant sizeclasstable.h includes from mem/ files. --- src/snmalloc/ds/ds.h | 1 + src/snmalloc/{mem => ds}/sizeclasstable.h | 30 +---------------- src/snmalloc/ds_core/ds_core.h | 3 ++ src/snmalloc/ds_core/sizeclassstatic.h | 41 +++++++++++++++++++++++ src/snmalloc/mem/backend_concept.h | 1 - src/snmalloc/mem/corealloc.h | 1 - src/snmalloc/mem/mem.h | 1 - src/snmalloc/mem/metadata.h | 1 - src/snmalloc/mem/remotecache.h | 1 - src/snmalloc/mem/secondary/gwp_asan.h | 2 +- 10 files changed, 47 insertions(+), 35 deletions(-) rename src/snmalloc/{mem => ds}/sizeclasstable.h (94%) create mode 100644 src/snmalloc/ds_core/sizeclassstatic.h diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index 47f097fc0..4ca68a410 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -10,3 +10,4 @@ #include "entropy.h" #include "mpmcstack.h" #include "pagemap.h" +#include "sizeclasstable.h" diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/ds/sizeclasstable.h similarity index 94% rename from src/snmalloc/mem/sizeclasstable.h rename to src/snmalloc/ds/sizeclasstable.h index d31a1f692..e1eaa2044 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/ds/sizeclasstable.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/ds.h" +#include "../pal/pal.h" /** * This file contains all the code for transforming transforming sizes to @@ -15,25 +15,8 @@ namespace snmalloc { - using smallsizeclass_t = size_t; using chunksizeclass_t = size_t; - static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) - { - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - auto sc = static_cast( - bits::to_exp_mant_const(size)); - - SNMALLOC_ASSERT(sc == static_cast(sc)); - - return sc; - } - - constexpr size_t NUM_SMALL_SIZECLASSES = - size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE) + 1; - // Large classes range from [MAX_SMALL_SIZECLASS_SIZE, ADDRESS_SPACE). constexpr size_t NUM_LARGE_CLASSES = DefaultPal::address_bits - MAX_SMALL_SIZECLASS_BITS; @@ -457,17 +440,6 @@ namespace snmalloc constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup(); - /** - * @brief Returns true if the size is a small sizeclass. Note that - * 0 is not considered a small sizeclass. - */ - constexpr bool is_small_sizeclass(size_t size) - { - // Perform the - 1 on size, so that zero wraps around and ends up on - // slow path. - return (size - 1) < sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1); - } - constexpr smallsizeclass_t size_to_sizeclass(size_t size) { if (SNMALLOC_LIKELY(is_small_sizeclass(size))) diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index fc9bd5be0..0eec777be 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -16,4 +16,7 @@ #include "mitigations.h" #include "ptrwrap.h" #include "redblacktree.h" +#include "sizeclassstatic.h" +#include "tid.h" + #include "tid.h" \ No newline at end of file diff --git a/src/snmalloc/ds_core/sizeclassstatic.h b/src/snmalloc/ds_core/sizeclassstatic.h new file mode 100644 index 000000000..bd1c975a6 --- /dev/null +++ b/src/snmalloc/ds_core/sizeclassstatic.h @@ -0,0 +1,41 @@ +#pragma once + +#include "allocconfig.h" +#include "bits.h" + +namespace snmalloc +{ + using smallsizeclass_t = size_t; + + static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) + { + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + auto sc = static_cast( + bits::to_exp_mant_const(size)); + + SNMALLOC_ASSERT(sc == static_cast(sc)); + + return sc; + } + + constexpr size_t NUM_SMALL_SIZECLASSES = + size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE) + 1; + + static constexpr size_t sizeclass_to_size_const(smallsizeclass_t sc) + { + return bits::from_exp_mant(sc); + } + + /** + * @brief Returns true if the size is a small sizeclass. Note that + * 0 is not considered a small sizeclass. + */ + constexpr bool is_small_sizeclass(size_t size) + { + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + return (size - 1) < sizeclass_to_size_const(NUM_SMALL_SIZECLASSES - 1); + } +} // namespace snmalloc diff --git a/src/snmalloc/mem/backend_concept.h b/src/snmalloc/mem/backend_concept.h index 9e0a21e77..524b40b9a 100644 --- a/src/snmalloc/mem/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -2,7 +2,6 @@ #ifdef __cpp_concepts # include "../ds/ds.h" -# include "sizeclasstable.h" # include diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 5ec7bf1f3..4d1926885 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -6,7 +6,6 @@ #include "metadata.h" #include "pool.h" #include "remotecache.h" -#include "sizeclasstable.h" #include "snmalloc/stl/new.h" #include "ticker.h" diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index 3e9212635..aa8b4eb7a 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -11,5 +11,4 @@ #include "pooled.h" #include "remoteallocator.h" #include "remotecache.h" -#include "sizeclasstable.h" #include "ticker.h" diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index 0284e8a5d..e753f125c 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -2,7 +2,6 @@ #include "../ds/ds.h" #include "freelist.h" -#include "sizeclasstable.h" #include "snmalloc/stl/new.h" namespace snmalloc diff --git a/src/snmalloc/mem/remotecache.h b/src/snmalloc/mem/remotecache.h index 045ab141b..c92c99a65 100644 --- a/src/snmalloc/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -5,7 +5,6 @@ #include "freelist.h" #include "metadata.h" #include "remoteallocator.h" -#include "sizeclasstable.h" #include "snmalloc/stl/array.h" #include "snmalloc/stl/atomic.h" diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index 178a3eeee..897bbb17f 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -2,7 +2,7 @@ #include "gwp_asan/guarded_pool_allocator.h" #include "snmalloc/ds_core/defines.h" -#include "snmalloc/mem/sizeclasstable.h" +#include "snmalloc/ds/sizeclasstable.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif From 07a0ee1b997632f422d72134d3d3fa74ca59a460 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 17:59:06 +0000 Subject: [PATCH 03/24] Make smallsizeclass_t a wrapper type Convert smallsizeclass_t from a size_t typedef to a struct with: - Explicit construction from size_t (no implicit conversion FROM) - Implicit conversion to size_t (for array indexing, comparisons) - Pre/post increment operators Provides type safety: alloc(size_t) (byte size) cannot be confused with alloc(smallsizeclass_t) (sizeclass index). Update all iteration sites to use explicit construction. --- src/snmalloc/ds/sizeclasstable.h | 14 +++---- src/snmalloc/ds_core/sizeclassstatic.h | 48 ++++++++++++++++++---- src/snmalloc/mem/corealloc.h | 7 ++-- src/test/func/malloc/malloc.cc | 22 +++++----- src/test/func/release-rounding/rounding.cc | 5 ++- src/test/func/sizeclass/sizeclass.cc | 6 +-- 6 files changed, 66 insertions(+), 36 deletions(-) diff --git a/src/snmalloc/ds/sizeclasstable.h b/src/snmalloc/ds/sizeclasstable.h index e1eaa2044..e09445b31 100644 --- a/src/snmalloc/ds/sizeclasstable.h +++ b/src/snmalloc/ds/sizeclasstable.h @@ -80,7 +80,7 @@ namespace snmalloc constexpr smallsizeclass_t as_small() { SNMALLOC_ASSERT(is_small()); - return value & (TAG - 1); + return smallsizeclass_t(value & (TAG - 1)); } constexpr chunksizeclass_t as_large() @@ -181,7 +181,7 @@ namespace snmalloc { size_t max_capacity = 0; - for (sizeclass_compress_t sizeclass = 0; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { @@ -213,7 +213,7 @@ namespace snmalloc // Get maximum precision to calculate largest division range. DIV_MULT_SHIFT = bits::BITS - bits::next_pow2_bits_const(max_capacity); - for (sizeclass_compress_t sizeclass = 0; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { @@ -417,7 +417,7 @@ namespace snmalloc sizeclass_compress_t sizeclass = 0; for (; sizeclass < minimum_class; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + for (; curr <= sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { table[sizeclass_lookup_index(curr)] = minimum_class; @@ -426,7 +426,7 @@ namespace snmalloc for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + for (; curr <= sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { auto i = sizeclass_lookup_index(curr); @@ -446,13 +446,13 @@ namespace snmalloc { auto index = sizeclass_lookup_index(size); SNMALLOC_ASSERT(index < sizeclass_lookup_size); - return sizeclass_lookup.table[index]; + return smallsizeclass_t(sizeclass_lookup.table[index]); } // Check this is not called on large sizes. SNMALLOC_ASSERT(size == 0); // Map size == 0 to the first sizeclass. - return 0; + return smallsizeclass_t(0); } /** diff --git a/src/snmalloc/ds_core/sizeclassstatic.h b/src/snmalloc/ds_core/sizeclassstatic.h index bd1c975a6..eba2181d4 100644 --- a/src/snmalloc/ds_core/sizeclassstatic.h +++ b/src/snmalloc/ds_core/sizeclassstatic.h @@ -5,23 +5,54 @@ namespace snmalloc { - using smallsizeclass_t = size_t; + /** + * A wrapper type for small sizeclass indices. + * + * Implicitly converts TO size_t (for array indexing, comparisons, etc.) + * but does NOT implicitly convert FROM size_t — construction must be + * explicit via smallsizeclass_t(value). + */ + struct smallsizeclass_t + { + size_t raw{0}; + + constexpr smallsizeclass_t() = default; + + explicit constexpr smallsizeclass_t(size_t v) : raw(v) {} + + /// Implicit conversion to size_t. + constexpr operator size_t() const + { + return raw; + } + + /// Pre-increment. + constexpr smallsizeclass_t& operator++() + { + ++raw; + return *this; + } + + /// Post-increment. + constexpr smallsizeclass_t operator++(int) + { + auto tmp = *this; + ++raw; + return tmp; + } + }; static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - auto sc = static_cast( + return smallsizeclass_t( bits::to_exp_mant_const(size)); - - SNMALLOC_ASSERT(sc == static_cast(sc)); - - return sc; } constexpr size_t NUM_SMALL_SIZECLASSES = - size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE) + 1; + size_t(size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE)) + 1; static constexpr size_t sizeclass_to_size_const(smallsizeclass_t sc) { @@ -36,6 +67,7 @@ namespace snmalloc { // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - return (size - 1) < sizeclass_to_size_const(NUM_SMALL_SIZECLASSES - 1); + return (size - 1) < + sizeclass_to_size_const(smallsizeclass_t(NUM_SMALL_SIZECLASSES - 1)); } } // namespace snmalloc diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 4d1926885..8da1748e8 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -606,8 +606,7 @@ namespace snmalloc { // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (SNMALLOC_LIKELY( - (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) + if (SNMALLOC_LIKELY(is_small_sizeclass(size))) { // Small allocations are more likely. Improve // branch prediction by placing this case first. @@ -1399,7 +1398,7 @@ namespace snmalloc auto& key = freelist::Object::key_root; - for (size_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) + for (smallsizeclass_t i; i < NUM_SMALL_SIZECLASSES; i++) { if (small_fast_free_lists[i].empty()) continue; @@ -1420,7 +1419,7 @@ namespace snmalloc local_state, get_trunc_id()); // We may now have unused slabs, return to the global allocator. - for (smallsizeclass_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { dealloc_local_slabs(sizeclass); diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index b3ab963cd..777417ca8 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -230,7 +230,7 @@ int main(int argc, char** argv) check_result(too_big_size, 1, our_malloc(too_big_size), ENOMEM, true); errno = SUCCESS; - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); START_TEST("malloc: {}", size); @@ -247,7 +247,7 @@ int main(int argc, char** argv) test_calloc(1, too_big_size, ENOMEM, true); errno = SUCCESS; - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); @@ -268,13 +268,13 @@ int main(int argc, char** argv) // Check realloc(nullptr,0) behaves like malloc(1) test_realloc(nullptr, 0, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < NUM_SMALL_SIZECLASSES; sc2++) + for (smallsizeclass_t sc2(0); sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); test_realloc(our_malloc(size), size2, SUCCESS, false); @@ -284,13 +284,13 @@ int main(int argc, char** argv) test_realloc(our_malloc(size), 0, SUCCESS, true); } - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); INFO("size1: {}, size2:{}\n", size, size2); @@ -311,7 +311,7 @@ int main(int argc, char** argv) // Check overflow with alignment taking it round to 0. test_memalign(1 - align, align, ENOMEM, true); - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES - 6; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES - 6; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); @@ -324,14 +324,14 @@ int main(int argc, char** argv) } test_reallocarray(nullptr, 1, 0, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_reallocarray(our_malloc(size), 1, size, SUCCESS, false); test_reallocarray(our_malloc(size), 1, 0, SUCCESS, false); test_reallocarray(nullptr, 1, size, SUCCESS, false); test_reallocarray(our_malloc(size), 1, too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); test_reallocarray(our_malloc(size), 1, size2, SUCCESS, false); @@ -342,7 +342,7 @@ int main(int argc, char** argv) test_reallocarr((size_t)~0, 1, 0, SUCCESS, false); test_reallocarr((size_t)~0, 1, 16, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_reallocarr(size, 1, size, SUCCESS, false); @@ -354,7 +354,7 @@ int main(int argc, char** argv) EXPECT(r == ENOMEM, "expected failure on allocation\n"); our_free(p); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); START_TEST("size1: {}, size2:{}", size, size2); diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index f9541331c..51b6e0b41 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -17,9 +17,10 @@ int main(int argc, char** argv) bool failed = false; - for (size_t size_class = 0; size_class < NUM_SMALL_SIZECLASSES; size_class++) + for (smallsizeclass_t size_class; size_class < NUM_SMALL_SIZECLASSES; + size_class++) { - size_t rsize = sizeclass_to_size((uint8_t)size_class); + size_t rsize = sizeclass_to_size(size_class); size_t max_offset = sizeclass_to_slab_size(size_class); sizeclass_t sc = sizeclass_t::from_small_class(size_class); for (size_t offset = 0; offset < max_offset; offset++) diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 836c62111..3b02003b0 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -17,9 +17,7 @@ void test_align_size() SNMALLOC_CHECK(snmalloc::aligned_size(128, 160) == 256); - for (size_t size = 1; - size < snmalloc::sizeclass_to_size(snmalloc::NUM_SMALL_SIZECLASSES - 1); - size++) + for (size_t size = 1; snmalloc::is_small_sizeclass(size); size++) { size_t rsize = snmalloc::round_size(size); @@ -85,7 +83,7 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::smallsizeclass_t sz = 0; sz < snmalloc::NUM_SMALL_SIZECLASSES; + for (snmalloc::smallsizeclass_t sz(0); sz < snmalloc::NUM_SMALL_SIZECLASSES; sz++) { if ( From 4a9d5b2464d8e18b576d50e97165d94fc7c4b956 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 18:03:36 +0000 Subject: [PATCH 04/24] Add alloc(smallsizeclass_t) fast path and libc entries Add alloc(smallsizeclass_t) overload to corealloc and globalalloc that skips the dynamic sizeclass lookup. Refactor small_alloc to accept (smallsizeclass_t, size_t) and keep the single-arg (size_t) version as a forwarding overload. Update alloc() to use compile-time sizeclass when the size is a known small sizeclass. Add libc::malloc_small(smallsizeclass_t) and libc::malloc_small_zero(smallsizeclass_t) as non-template entry points. --- src/snmalloc/global/globalalloc.h | 25 +++++++++++++++++++++-- src/snmalloc/global/libc.h | 19 +++++++++++++++++ src/snmalloc/mem/corealloc.h | 34 +++++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 110051e2f..79d6a98b7 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -324,8 +324,18 @@ namespace snmalloc template SNMALLOC_FAST_PATH_INLINE void* alloc() { - return ThreadAlloc::get().alloc( - aligned_size(align, size)); + constexpr size_t sz = aligned_size(align, size); + if constexpr (is_small_sizeclass(sz)) + { + constexpr auto sc = size_to_sizeclass_const(sz); + return ThreadAlloc::get().template alloc( + sc); + } + else + { + return ThreadAlloc::get().template alloc( + sz); + } } template @@ -335,6 +345,17 @@ namespace snmalloc aligned_size(align, size)); } + /** + * Allocate a block for a known small sizeclass. + * The sizeclass can be computed at compile time with size_to_sizeclass_const. + */ + template + SNMALLOC_FAST_PATH_INLINE void* alloc(smallsizeclass_t sizeclass) + { + return ThreadAlloc::get().template alloc( + sizeclass); + } + template SNMALLOC_FAST_PATH_INLINE void* alloc_aligned(size_t align, size_t size) { diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index 726f61190..de0c02976 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -29,6 +29,25 @@ namespace snmalloc::libc return snmalloc::alloc(size); } + /** + * Allocate for a pre-computed small sizeclass. + * Use is_small_sizeclass() + size_to_sizeclass_const() to get the class. + */ + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + malloc_small(smallsizeclass_t sizeclass) + { + return snmalloc::alloc(sizeclass); + } + + /** + * Allocate zeroed memory for a pre-computed small sizeclass. + */ + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + malloc_small_zero(smallsizeclass_t sizeclass) + { + return snmalloc::alloc(sizeclass); + } + SNMALLOC_FAST_PATH_INLINE void free(void* ptr) { dealloc(ptr); diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 8da1748e8..ba5fa1278 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -610,18 +610,34 @@ namespace snmalloc { // Small allocations are more likely. Improve // branch prediction by placing this case first. - return small_alloc(size); + return small_alloc( + size_to_sizeclass(size), size); } return alloc_not_small(size, this); } /** - * Fast allocation for small objects. + * Allocates a block of memory for a known small sizeclass. + * Callers can pre-compute the sizeclass (e.g. at compile time with + * size_to_sizeclass_const) and avoid the dynamic sizeclass lookup. + */ + template + SNMALLOC_FAST_PATH ALLOCATOR void* + alloc(smallsizeclass_t sizeclass) noexcept(noexcept(Conts::failure(0))) + { + return small_alloc( + sizeclass, sizeclass_to_size(sizeclass)); + } + + /** + * Fast allocation for small objects, with pre-computed sizeclass. */ template SNMALLOC_FAST_PATH void* - small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) + small_alloc( + smallsizeclass_t sizeclass, + size_t size) noexcept(noexcept(Conts::failure(0))) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { @@ -629,7 +645,6 @@ namespace snmalloc }; auto& key = freelist::Object::key_root; - smallsizeclass_t sizeclass = size_to_sizeclass(size); auto* fl = &small_fast_free_lists[sizeclass]; if (SNMALLOC_LIKELY(!fl->empty())) { @@ -651,6 +666,17 @@ namespace snmalloc size); } + /** + * Fast allocation for small objects from a byte size. + * Computes the sizeclass and delegates to the two-arg version. + */ + template + SNMALLOC_FAST_PATH void* + small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) + { + return small_alloc(size_to_sizeclass(size), size); + } + /** * Allocation that are larger that will result in an allocation directly * from the backend. This additionally checks for 0 size allocations and From d3a39148eb4f1e6ae1cff30bdeacc2fd337bf121 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 18:07:27 +0000 Subject: [PATCH 05/24] Move pool.h/pooled.h from mem/ to ds/ These are general-purpose data structures, not allocator-specific. Moving them reduces the dependency surface of mem/. --- src/snmalloc/ds/ds.h | 2 ++ src/snmalloc/{mem => ds}/pool.h | 0 src/snmalloc/{mem => ds}/pooled.h | 2 +- src/snmalloc/global/globalalloc.h | 10 +++++---- src/snmalloc/global/libc.h | 34 +++++++++++++++++++------------ src/snmalloc/mem/corealloc.h | 2 +- src/snmalloc/mem/mem.h | 4 ++-- 7 files changed, 33 insertions(+), 21 deletions(-) rename src/snmalloc/{mem => ds}/pool.h (100%) rename src/snmalloc/{mem => ds}/pooled.h (94%) diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index 4ca68a410..9c9cd0a49 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -10,4 +10,6 @@ #include "entropy.h" #include "mpmcstack.h" #include "pagemap.h" +#include "pool.h" +#include "pooled.h" #include "sizeclasstable.h" diff --git a/src/snmalloc/mem/pool.h b/src/snmalloc/ds/pool.h similarity index 100% rename from src/snmalloc/mem/pool.h rename to src/snmalloc/ds/pool.h diff --git a/src/snmalloc/mem/pooled.h b/src/snmalloc/ds/pooled.h similarity index 94% rename from src/snmalloc/mem/pooled.h rename to src/snmalloc/ds/pooled.h index ca4e94101..d64716783 100644 --- a/src/snmalloc/mem/pooled.h +++ b/src/snmalloc/ds/pooled.h @@ -1,7 +1,7 @@ #pragma once #include "../ds/ds.h" -#include "backend_concept.h" +#include "../mem/backend_concept.h" namespace snmalloc { diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 79d6a98b7..7499feea5 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -363,12 +363,13 @@ namespace snmalloc aligned_size(align, size)); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) { ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + dealloc(void* p, size_t size) { check_size(p, size); ThreadAlloc::get().dealloc(p); @@ -381,14 +382,15 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size, size_t align) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + dealloc(void* p, size_t size, size_t align) { auto rsize = aligned_size(align, size); check_size(p, rsize); ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void debug_teardown() + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void debug_teardown() { return ThreadAlloc::teardown(); } diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index de0c02976..1abe40665 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -19,12 +19,12 @@ namespace snmalloc::libc return err; } - inline void* __malloc_end_pointer(void* ptr) + SNMALLOC_USED_FUNCTION inline void* __malloc_end_pointer(void* ptr) { return snmalloc::external_pointer(ptr); } - SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) { return snmalloc::alloc(size); } @@ -48,23 +48,25 @@ namespace snmalloc::libc return snmalloc::alloc(sizeclass); } - SNMALLOC_FAST_PATH_INLINE void free(void* ptr) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void free(void* ptr) { dealloc(ptr); } - SNMALLOC_FAST_PATH_INLINE void free_sized(void* ptr, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + free_sized(void* ptr, size_t size) { dealloc(ptr, size); } - SNMALLOC_FAST_PATH_INLINE void + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void free_aligned_sized(void* ptr, size_t alignment, size_t size) { dealloc(ptr, size, alignment); } - SNMALLOC_FAST_PATH_INLINE void* calloc(size_t nmemb, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + calloc(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -75,7 +77,8 @@ namespace snmalloc::libc return alloc(sz); } - SNMALLOC_FAST_PATH_INLINE void* realloc(void* ptr, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + realloc(void* ptr, size_t size) { // Glibc treats // realloc(p, 0) as free(p) @@ -124,12 +127,13 @@ namespace snmalloc::libc return p; } - inline size_t malloc_usable_size(const void* ptr) + SNMALLOC_USED_FUNCTION inline size_t malloc_usable_size(const void* ptr) { return alloc_size(ptr); } - inline void* reallocarray(void* ptr, size_t nmemb, size_t size) + SNMALLOC_USED_FUNCTION inline void* + reallocarray(void* ptr, size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -140,7 +144,8 @@ namespace snmalloc::libc return realloc(ptr, sz); } - inline int reallocarr(void* ptr_, size_t nmemb, size_t size) + SNMALLOC_USED_FUNCTION inline int + reallocarr(void* ptr_, size_t nmemb, size_t size) { int err = errno; bool overflow = false; @@ -175,7 +180,8 @@ namespace snmalloc::libc return 0; } - inline void* memalign(size_t alignment, size_t size) + SNMALLOC_USED_FUNCTION inline void* + memalign(size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY(alignment == 0 || !bits::is_pow2(alignment))) { @@ -185,12 +191,14 @@ namespace snmalloc::libc return alloc_aligned(alignment, size); } - inline void* aligned_alloc(size_t alignment, size_t size) + SNMALLOC_USED_FUNCTION inline void* + aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } - inline int posix_memalign(void** memptr, size_t alignment, size_t size) + SNMALLOC_USED_FUNCTION inline int + posix_memalign(void** memptr, size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY( (alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment)))) diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index ba5fa1278..9bcee944d 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -4,7 +4,7 @@ #include "check_init.h" #include "freelist.h" #include "metadata.h" -#include "pool.h" +#include "../ds/pool.h" #include "remotecache.h" #include "snmalloc/stl/new.h" #include "ticker.h" diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index aa8b4eb7a..6ee3caea6 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -7,8 +7,8 @@ #include "entropy.h" #include "freelist.h" #include "metadata.h" -#include "pool.h" -#include "pooled.h" +#include "../ds/pool.h" +#include "../ds/pooled.h" #include "remoteallocator.h" #include "remotecache.h" #include "ticker.h" From 13ed7cd987354d524d96c586f547ed3eb6803df9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 18:13:09 +0000 Subject: [PATCH 06/24] Add external_pointer libc functions Add __malloc_start_pointer() and __malloc_last_byte_pointer() to libc.h, alongside the existing __malloc_end_pointer(). These provide the external_pointer functionality through the libc API surface. --- src/snmalloc/global/libc.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index 1abe40665..c1c389f2e 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -24,6 +24,16 @@ namespace snmalloc::libc return snmalloc::external_pointer(ptr); } + SNMALLOC_USED_FUNCTION inline void* __malloc_start_pointer(void* ptr) + { + return snmalloc::external_pointer(ptr); + } + + SNMALLOC_USED_FUNCTION inline void* __malloc_last_byte_pointer(void* ptr) + { + return snmalloc::external_pointer(ptr); + } + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) { return snmalloc::alloc(size); From ac468b3a8aa32157c8c3a6e137b8076a3c740192 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 18:22:47 +0000 Subject: [PATCH 07/24] Create test library infrastructure Add snmalloc_testlib.h and snmalloc_testlib.cc. CMake build_test_library() creates a static library per flavour linked to all tests. Add SNMALLOC_BUILD_TESTING=OFF to Bazel, src/test/*.cc to Bazel filegroup, and sanitizer flags to build_test_library. --- CMakeLists.txt | 17 +++ src/test/snmalloc_testlib.cc | 214 +++++++++++++++++++++++++++++++++++ src/test/snmalloc_testlib.h | 184 ++++++++++++++++++++++++++++++ 3 files changed, 415 insertions(+) create mode 100644 src/test/snmalloc_testlib.cc create mode 100644 src/test/snmalloc_testlib.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e21d6bd71..1fea1b75d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -501,6 +501,10 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) add_warning_flags(${TESTNAME}) target_link_libraries(${TESTNAME} snmalloc) + # Link the pre-compiled test library into all tests. For tests + # that include snmalloc.h directly the linker will simply discard + # the duplicate inline definitions (ODR-safe). + target_link_libraries(${TESTNAME} snmalloc-testlib-${TAG}) target_compile_definitions(${TESTNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") if (NOT DEFINES STREQUAL " ") @@ -538,6 +542,18 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endforeach() endfunction() + function(build_test_library TAG DEFINES) + set(LIBNAME snmalloc-testlib-${TAG}) + add_library(${LIBNAME} STATIC src/test/snmalloc_testlib.cc) + target_link_libraries(${LIBNAME} PRIVATE snmalloc) + set_target_properties(${LIBNAME} PROPERTIES INTERFACE_LINK_LIBRARIES "") + target_compile_definitions(${LIBNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") + if (NOT DEFINES STREQUAL " ") + target_compile_definitions(${LIBNAME} PRIVATE ${DEFINES}) + endif() + add_warning_flags(${LIBNAME}) + endfunction() + if(NOT (DEFINED SNMALLOC_LINKER_FLAVOUR) OR ("${SNMALLOC_LINKER_FLAVOUR}" MATCHES "^$")) # Linker not specified externally; probe to see if we can make lld work set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld -Wl,--icf=all) @@ -680,6 +696,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(DEFINES " ") endif() + build_test_library(${FLAVOUR} ${DEFINES}) make_tests(${FLAVOUR} ${DEFINES}) endforeach() endif() diff --git a/src/test/snmalloc_testlib.cc b/src/test/snmalloc_testlib.cc new file mode 100644 index 000000000..d5f8e31df --- /dev/null +++ b/src/test/snmalloc_testlib.cc @@ -0,0 +1,214 @@ +/** + * @file snmalloc_testlib.cc + * @brief Single translation unit that compiles the full snmalloc allocator + * and provides symbols for test-library consumers. + * + * Built once per flavour (fast / check) into a static library that tests + * link against, avoiding redundant recompilation of the allocator in every + * test TU. + * + * Three kinds of symbols are provided: + * + * A. Non-templated inline functions (dealloc, debug_teardown, libc::*) + * are marked SNMALLOC_USED_FUNCTION in their declarations, so weak/COMDAT + * symbols are emitted in every TU that includes them; the linker resolves + * test references to the copies in this archive. + * + * B. Templates on user-visible params only (alloc): + * explicit instantiation definitions force emission of strong symbols. + * + * C. Config-templated functions: non-template overloads that call through + * to the template version. These are distinct functions (not + * ODR-conflicting with the templates). + */ + +#include + +namespace snmalloc +{ + template + void* alloc(size_t size) + { + if constexpr (zero_mem == ZeroMem::YesZero) + { + return alloc(size); + } + else + { + return alloc(size); + } + } + + template void* alloc(size_t size); + template void* alloc(size_t size); + + // -- C: Non-template wrappers for Config-templated functions ------------- + + size_t alloc_size(const void* p) + { + return alloc_size(p); + } + + size_t remaining_bytes(address_t p) + { + return remaining_bytes(p); + } + + bool is_owned(void* p) + { + return is_owned(p); + } + + void debug_check_empty(bool* result) + { + debug_check_empty(result); + } + + void debug_in_use(size_t count) + { + debug_in_use(count); + } + + void cleanup_unused() + { + cleanup_unused(); + } + + // -- Opaque scoped allocator (inherits from ScopedAllocator<>) ----------- + + struct TestScopedAllocator : public ScopedAllocator<> + {}; + + TestScopedAllocator* create_scoped_allocator() + { + return new TestScopedAllocator(); + } + + void destroy_scoped_allocator(TestScopedAllocator* p) + { + delete p; + } + + void* scoped_alloc(TestScopedAllocator* a, size_t size) + { + return a->alloc->alloc(size); + } + + // ScopedAllocHandle is declared in snmalloc_testlib.h; define methods here. + struct ScopedAllocHandle + { + TestScopedAllocator* ptr; + ScopedAllocHandle(); + ~ScopedAllocHandle(); + ScopedAllocHandle(const ScopedAllocHandle&) = delete; + ScopedAllocHandle& operator=(const ScopedAllocHandle&) = delete; + void* alloc(size_t size); + void dealloc(void* p); + void dealloc(void* p, size_t size); + ScopedAllocHandle* operator->(); + const ScopedAllocHandle* operator->() const; + }; + + ScopedAllocHandle::ScopedAllocHandle() : ptr(create_scoped_allocator()) {} + + ScopedAllocHandle::~ScopedAllocHandle() + { + destroy_scoped_allocator(ptr); + } + + void* ScopedAllocHandle::alloc(size_t size) + { + return scoped_alloc(ptr, size); + } + + void ScopedAllocHandle::dealloc(void* p) + { + snmalloc::dealloc(p); + } + + void ScopedAllocHandle::dealloc(void* p, size_t size) + { + snmalloc::dealloc(p, size); + } + + ScopedAllocHandle* ScopedAllocHandle::operator->() + { + return this; + } + + const ScopedAllocHandle* ScopedAllocHandle::operator->() const + { + return this; + } + + ScopedAllocHandle get_scoped_allocator() + { + return {}; + } + + size_t max_small_sizeclass_bits() + { + return MAX_SMALL_SIZECLASS_BITS; + } + + // -- PAL/AAL wrappers ---------------------------------------------------- + + size_t pal_address_bits() + { + return DefaultPal::address_bits; + } + + uint64_t pal_tick() + { + return DefaultPal::tick(); + } + + void pal_pause() + { + Aal::pause(); + } + + // -- Force emission of inline functions for MSVC (where USED_FUNCTION is + // empty). Taking the address of each inline function forces the compiler to + // emit a definition in this TU, making it available to testlib consumers. + + SNMALLOC_USED_FUNCTION static const volatile void* force_emit_global[] = { + reinterpret_cast( + static_cast(&dealloc)), + reinterpret_cast( + static_cast(&dealloc)), + reinterpret_cast( + static_cast(&dealloc)), + reinterpret_cast(&debug_teardown), + }; + + namespace libc + { + SNMALLOC_USED_FUNCTION static const volatile void* force_emit_libc[] = { + reinterpret_cast(&__malloc_start_pointer), + reinterpret_cast(&__malloc_last_byte_pointer), + reinterpret_cast(&__malloc_end_pointer), + reinterpret_cast(&malloc), + reinterpret_cast(&free), + reinterpret_cast( + static_cast(&calloc)), + reinterpret_cast( + static_cast(&realloc)), + reinterpret_cast(&malloc_usable_size), + reinterpret_cast(&memalign), + reinterpret_cast(&aligned_alloc), + reinterpret_cast(&posix_memalign), + reinterpret_cast(&malloc_small), + reinterpret_cast(&malloc_small_zero), + }; + } // namespace libc +} // namespace snmalloc + +// -- override/malloc.cc symbols with testlib_ prefix ----------------------- +#undef SNMALLOC_NO_REALLOCARRAY +#undef SNMALLOC_NO_REALLOCARR +#define SNMALLOC_NAME_MANGLE(a) testlib_##a +#include + +// malloc-extensions (unprefixed - get_malloc_info_v1) +#include diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h new file mode 100644 index 000000000..32cf2c2ea --- /dev/null +++ b/src/test/snmalloc_testlib.h @@ -0,0 +1,184 @@ +#pragma once +/** + * @file snmalloc_testlib.h + * @brief Thin test-library header that replaces for + * tests that can be linked against a pre-compiled static library. + * + * Does NOT include snmalloc_core.h — only forward-declares the minimal types + * needed for the API surface. Tests that need snmalloc_core types (bits::*, + * sizeclass, Pal, etc.) should include themselves + * before this header. + * + * Config-templated functions are provided as non-template overloads + * (distinct from the templates, no ODR conflict) defined in testlib.cc. + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace snmalloc +{ + // Forward declarations sufficient for the API surface. + // address_t: uintptr_t on all non-CHERI platforms. + using address_t = uintptr_t; + + template + class DefaultConts; + // Uninit / Zero are usable as template arguments even without the full + // definition of DefaultConts (only needed inside the allocator itself). + using Uninit = DefaultConts; + using Zero = DefaultConts; + + enum Boundary + { + Start, + End, + OnePastEnd + }; + + // -- Non-template functions (symbol forced in testlib.cc) ---------------- + void dealloc(void* p); + void dealloc(void* p, size_t size); + void dealloc(void* p, size_t size, size_t align); + + template + inline void dealloc(void* p) + { + dealloc(p, size); + } + + void debug_teardown(); + + // -- Non-template wrappers for Config-templated functions ---------------- + size_t alloc_size(const void* p); + size_t remaining_bytes(address_t p); + bool is_owned(void* p); + void debug_check_empty(bool* result = nullptr); + void debug_in_use(size_t count); + void cleanup_unused(); + + // -- Opaque scoped allocator --------------------------------------------- + // TestScopedAllocator inherits from ScopedAllocator<> in testlib.cc. + // Forward-declared here; usable through ScopedAllocHandle. + struct TestScopedAllocator; + TestScopedAllocator* create_scoped_allocator(); + void destroy_scoped_allocator(TestScopedAllocator*); + void* scoped_alloc(TestScopedAllocator*, size_t size); + + struct ScopedAllocHandle + { + TestScopedAllocator* ptr; + + ScopedAllocHandle(); + ~ScopedAllocHandle(); + ScopedAllocHandle(const ScopedAllocHandle&) = delete; + ScopedAllocHandle& operator=(const ScopedAllocHandle&) = delete; + void* alloc(size_t size); + + void dealloc(void* p); + void dealloc(void* p, size_t size); + + ScopedAllocHandle* operator->(); + const ScopedAllocHandle* operator->() const; + }; + + ScopedAllocHandle get_scoped_allocator(); + + // -- Constants exposed from allocconfig.h -------------------------------- + size_t max_small_sizeclass_bits(); + + // -- PAL/AAL wrappers (avoids needing pal.h in tests) -------------------- + size_t pal_address_bits(); + uint64_t pal_tick(); + void pal_pause(); + + // Simple pointer arithmetic (matches snmalloc::pointer_offset for void*). + // Defined inline so tests that don't include aal/address.h can still use it. + inline void* pointer_offset(void* base, size_t diff) + { + return static_cast(static_cast(base) + diff); + } + + template + void* alloc(size_t size); + extern template void* alloc(size_t); + extern template void* alloc(size_t); + + template + void* alloc_aligned(size_t align, size_t size); + + // -- libc namespace ------------------------------------------------------ + namespace libc + { + void* malloc(size_t size); + void* malloc_small(smallsizeclass_t sizeclass); + void* malloc_small_zero(smallsizeclass_t sizeclass); + void free(void* ptr); + void* calloc(size_t nmemb, size_t size); + void* realloc(void* ptr, size_t size); + size_t malloc_usable_size(const void* ptr); + void* memalign(size_t alignment, size_t size); + void* aligned_alloc(size_t alignment, size_t size); + int posix_memalign(void** memptr, size_t alignment, size_t size); + void* reallocarray(void* ptr, size_t nmemb, size_t size); + int reallocarr(void* ptr_, size_t nmemb, size_t size); + void* __malloc_start_pointer(void* ptr); + void* __malloc_last_byte_pointer(void* ptr); + void* __malloc_end_pointer(void* ptr); + } // namespace libc + + /** + * Compile-time size alloc. When the size (after alignment) is a small + * sizeclass the sizeclass is computed at compile time and the allocation + * goes straight to the sizeclass-based fast path. Otherwise falls back + * to the dynamic alloc. + */ + template + inline void* alloc() + { + if constexpr (is_small_sizeclass(size)) + { + constexpr auto sc = size_to_sizeclass_const(size); + if constexpr (zero_mem == ZeroMem::YesZero) + { + return libc::malloc_small_zero(sc); + } + else + { + return libc::malloc_small(sc); + } + } + else + { + return alloc(size); + } + } +} // namespace snmalloc + +// -- malloc-extensions struct and function ---------------------------------- +#include + +// -- override/malloc.cc functions with testlib_ prefix ---------------------- +#ifndef MALLOC_USABLE_SIZE_QUALIFIER +# define MALLOC_USABLE_SIZE_QUALIFIER +#endif +extern "C" +{ + void* testlib_malloc(size_t size); + void testlib_free(void* ptr); + void testlib_cfree(void* ptr); + void* testlib_calloc(size_t nmemb, size_t size); + size_t testlib_malloc_usable_size(MALLOC_USABLE_SIZE_QUALIFIER void* ptr); + size_t testlib_malloc_good_size(size_t size); + void* testlib_realloc(void* ptr, size_t size); + void* testlib_reallocarray(void* ptr, size_t nmemb, size_t size); + int testlib_reallocarr(void* ptr, size_t nmemb, size_t size); + void* testlib_memalign(size_t alignment, size_t size); + void* testlib_aligned_alloc(size_t alignment, size_t size); + int testlib_posix_memalign(void** memptr, size_t alignment, size_t size); +} From d4ff8935f2ad5e4117d7b01a6923ce60d346b9fb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 18:25:12 +0000 Subject: [PATCH 08/24] Convert batch 9a tests to use testlib Convert simple alloc/dealloc tests to use snmalloc_testlib.h instead of snmalloc.h: - func/first_operation - func/teardown - func/multi_atexit - func/multi_threadatexit Replace MAX_SMALL_SIZECLASS_BITS with max_small_sizeclass_bits(). Replace alloc with alloc. Add pal/pal.h include to testlib header for report_fatal_error. --- src/test/func/first_operation/first_operation.cc | 8 ++++---- src/test/func/teardown/teardown.cc | 8 ++++---- src/test/snmalloc_testlib.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index c28d381af..d375cbef0 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void alloc1(size_t size) @@ -47,14 +47,14 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r); } void calloc2(size_t size) { - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r, size); } @@ -109,7 +109,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::max_small_sizeclass_bits(); exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index b20d8b1cf..02129374d 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void trigger_teardown() @@ -59,7 +59,7 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { trigger_teardown(); - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r); } @@ -67,7 +67,7 @@ void calloc1(size_t size) void calloc2(size_t size) { trigger_teardown(); - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r, size); } @@ -124,7 +124,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::max_small_sizeclass_bits(); exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h index 32cf2c2ea..dc51ed67a 100644 --- a/src/test/snmalloc_testlib.h +++ b/src/test/snmalloc_testlib.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include From 6d68c71af4729758953799d6c2dfe6a64369eda2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 18:27:46 +0000 Subject: [PATCH 09/24] Convert perf tests to use testlib (batch 9d partial) Convert perf tests that use only the public API: - perf/startup - perf/contention - perf/large_alloc - perf/low_memory Replace DefaultPal::tick() with pal_tick(), Aal::pause() with pal_pause(). --- src/test/perf/contention/contention.cc | 9 ++++----- src/test/perf/large_alloc/large_alloc.cc | 2 +- src/test/perf/low_memory/low-memory.cc | 2 +- src/test/perf/startup/startup.cc | 13 ++++++------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 6bed00545..bca2a4889 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -5,8 +5,7 @@ #include #include -#include -#include +#include #include #include @@ -31,18 +30,18 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = DefaultPal::tick(); + start = snmalloc::pal_tick(); flag = true; } while (!flag) - Aal::pause(); + snmalloc::pal_pause(); f(id); prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = DefaultPal::tick(); + end = snmalloc::pal_tick(); } } diff --git a/src/test/perf/large_alloc/large_alloc.cc b/src/test/perf/large_alloc/large_alloc.cc index 2532b41e3..71cbdce80 100644 --- a/src/test/perf/large_alloc/large_alloc.cc +++ b/src/test/perf/large_alloc/large_alloc.cc @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index c9e973cc0..ee3f68884 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/src/test/perf/startup/startup.cc b/src/test/perf/startup/startup.cc index dfc6cb0c2..30fa23438 100644 --- a/src/test/perf/startup/startup.cc +++ b/src/test/perf/startup/startup.cc @@ -5,8 +5,7 @@ #include #include -#include -#include +#include #include #include @@ -31,18 +30,18 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = DefaultPal::tick(); + start = snmalloc::pal_tick(); flag = true; } while (!flag) - Aal::pause(); + snmalloc::pal_pause(); f(id); prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = DefaultPal::tick(); + end = snmalloc::pal_tick(); } } @@ -77,9 +76,9 @@ int main() ParallelTest test( [](size_t id) { - auto start = DefaultPal::tick(); + auto start = snmalloc::pal_tick(); snmalloc::dealloc(snmalloc::alloc(1)); - auto end = DefaultPal::tick(); + auto end = snmalloc::pal_tick(); counters[id] = end - start; }, nthreads); From 2fda2ebb817189375c79fc1b69c5fd44c16e1c88 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 20:25:44 +0000 Subject: [PATCH 10/24] Convert remaining testlib-only tests (batch 9b/9c/9d) Convert tests that use only the public API to snmalloc_testlib.h: - func/bits, func/memory, func/memory_usage, func/pool - func/protect_fork, func/redblack, func/statistics - perf/external_pointer, perf/lotsofthreads, perf/post_teardown - perf/singlethread Replace DefaultPal/Aal calls with pal_*() wrappers. Replace external_pointer<> with libc __malloc_*_pointer(). Replace alloc with alloc. Fix include order in ds/ds.h for pool.h/pooled.h dependencies. --- src/snmalloc/ds/ds.h | 2 +- src/test/func/bits/bits.cc | 2 +- src/test/func/memory/memory.cc | 40 +++++++++---------- src/test/func/memory_usage/memory_usage.cc | 9 ++--- src/test/func/pool/pool.cc | 12 ++---- src/test/func/protect_fork/protect_fork.cc | 2 +- src/test/func/redblack/redblack.cc | 4 +- src/test/func/statistics/stats.cc | 2 +- .../perf/external_pointer/externalpointer.cc | 6 +-- src/test/perf/lotsofthreads/lotsofthread.cc | 2 +- src/test/perf/post_teardown/post-teardown.cc | 6 +-- src/test/perf/singlethread/singlethread.cc | 26 ++++++------ 12 files changed, 53 insertions(+), 60 deletions(-) diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index 9c9cd0a49..4a512c975 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -10,6 +10,6 @@ #include "entropy.h" #include "mpmcstack.h" #include "pagemap.h" +#include "sizeclasstable.h" #include "pool.h" #include "pooled.h" -#include "sizeclasstable.h" diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index 0046516a3..01deaaeea 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -3,7 +3,7 @@ */ #include -#include +#include #include void test_ctz() diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 7c181a24a..7bfd4175e 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,9 +1,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -178,7 +178,7 @@ void test_calloc() memset(p, 0xFF, size); snmalloc::dealloc(p, size); - p = snmalloc::alloc(size); + p = snmalloc::alloc(size); for (size_t i = 0; i < size; i++) { @@ -239,11 +239,11 @@ void test_double_alloc() void test_external_pointer() { - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass_const(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { - size_t size = sizeclass_to_size(sc); + size_t size = sizeclass_to_size_const(sc); void* p1 = snmalloc::alloc(size); if (size != snmalloc::alloc_size(p1)) @@ -259,8 +259,8 @@ void test_external_pointer() for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); - void* p3 = snmalloc::external_pointer(p2); - void* p4 = snmalloc::external_pointer(p2); + void* p3 = snmalloc::libc::__malloc_start_pointer(p2); + void* p4 = snmalloc::libc::__malloc_last_byte_pointer(p2); if (p1 != p3) { if (p3 > p1 || snmalloc::is_owned(p1)) @@ -293,7 +293,7 @@ void test_external_pointer() void check_offset(void* base, void* interior) { - void* calced_base = snmalloc::external_pointer((void*)interior); + void* calced_base = snmalloc::libc::__malloc_start_pointer((void*)interior); if (calced_base != (void*)base) { if (calced_base > base || snmalloc::is_owned(base)) @@ -320,10 +320,10 @@ void test_external_pointer_large() { xoroshiro::p128r64 r; - constexpr size_t count_log = DefaultPal::address_bits > 32 ? 5 : 3; - constexpr size_t count = 1 << count_log; + const size_t count_log = pal_address_bits() > 32 ? 5 : 3; + const size_t count = size_t(1) << count_log; // Pre allocate all the objects - size_t* objects[count]; + size_t* objects[1 << 5]; // max possible count size_t total_size = 0; @@ -376,7 +376,7 @@ void test_external_pointer_dealloc_bug() for (size_t i = 0; i < count; i++) { - snmalloc::external_pointer(allocs[i]); + snmalloc::libc::__malloc_start_pointer(allocs[i]); } snmalloc::dealloc(allocs[0]); @@ -391,10 +391,10 @@ void test_external_pointer_stack() for (size_t i = 0; i < stack.size(); i++) { - if (snmalloc::external_pointer(&stack[i]) > &stack[i]) + if (snmalloc::libc::__malloc_start_pointer(&stack[i]) > &stack[i]) { std::cout << "Stack pointer: " << &stack[i] << " external pointer: " - << snmalloc::external_pointer(&stack[i]) << std::endl; + << snmalloc::libc::__malloc_start_pointer(&stack[i]) << std::endl; abort(); } } @@ -408,7 +408,7 @@ void test_alloc_16M() const size_t size = 16'000'000; void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -417,8 +417,8 @@ void test_calloc_16M() // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -430,8 +430,8 @@ void test_calloc_large_bug() // not a multiple of page size. const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; - void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -485,11 +485,11 @@ void test_static_sized_allocs() void test_remaining_bytes() { - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass_const(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { - auto size = sizeclass_to_size(sc); + auto size = sizeclass_to_size_const(sc); char* p = (char*)snmalloc::alloc(size); for (size_t offset = 0; offset < size; offset++) { diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 68fa179f7..036bcfe2f 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -4,12 +4,9 @@ */ #include #include +#include #include -#define SNMALLOC_NAME_MANGLE(a) our_##a -#include "../../../snmalloc/override/malloc-extensions.cc" -#include "../../../snmalloc/override/malloc.cc" - using namespace snmalloc; bool print_memory_usage() @@ -43,7 +40,7 @@ void add_n_allocs(size_t n) { while (true) { - auto p = our_malloc(1024); + auto p = testlib_malloc(1024); allocs.push_back(p); if (print_memory_usage()) { @@ -64,7 +61,7 @@ void remove_n_allocs(size_t n) if (allocs.empty()) return; auto p = allocs.back(); - our_free(p); + testlib_free(p); allocs.pop_back(); if (print_memory_usage()) { diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 6a2874215..8f11ff689 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -1,8 +1,8 @@ #include #include -#include -#include +#include #include +#include #include using namespace snmalloc; @@ -56,7 +56,7 @@ void test_alloc() auto ptr = PoolA::acquire(); SNMALLOC_CHECK(ptr != nullptr); // Pool allocations should not be visible to debug_check_empty. - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); PoolA::release(ptr); } @@ -242,13 +242,7 @@ void test_sort() int main(int argc, char** argv) { setup(); -#ifdef USE_SYSTEMATIC_TESTING - opt::Opt opt(argc, argv); - size_t seed = opt.is("--seed", 0); - Virtual::systematic_bump_ptr() += seed << 17; -#else UNUSED(argc, argv); -#endif test_double_alloc(); std::cout << "test_double_alloc passed" << std::endl; diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index 77227b202..4ab2d5a45 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -9,7 +9,7 @@ int main() # define SNMALLOC_PTHREAD_FORK_PROTECTION # include -# include +# include # include void simulate_allocation() diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 164a5978f..61fccb6d3 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -11,7 +11,9 @@ # define SNMALLOC_TRACING #endif // Redblack tree needs some libraries with trace enabled. -#include "snmalloc/snmalloc.h" +#include "test/snmalloc_testlib.h" + +#include struct NodeRef { diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index d66f060a1..cf3596748 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -1,5 +1,5 @@ #include -#include +#include #include template diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 76be4c708..df6f23ab3 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -1,6 +1,6 @@ -#include #include #include +#include #include #include @@ -19,7 +19,7 @@ namespace test { size_t rand = (size_t)r.next(); size_t offset = bits::clz(rand); - if constexpr (DefaultPal::address_bits > 32) + if (snmalloc::pal_address_bits() > 32) { if (offset > 30) offset = 30; @@ -80,7 +80,7 @@ namespace test size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); - void* calced_external = snmalloc::external_pointer(interior_ptr); + void* calced_external = snmalloc::libc::__malloc_start_pointer(interior_ptr); if (calced_external != external_ptr) { abort(); diff --git a/src/test/perf/lotsofthreads/lotsofthread.cc b/src/test/perf/lotsofthreads/lotsofthread.cc index 000af6534..9705dfff3 100644 --- a/src/test/perf/lotsofthreads/lotsofthread.cc +++ b/src/test/perf/lotsofthreads/lotsofthread.cc @@ -19,7 +19,7 @@ #include using namespace std; -#include +#include #define malloc snmalloc::libc::malloc #define free snmalloc::libc::free #define malloc_usable_size snmalloc::libc::malloc_usable_size diff --git a/src/test/perf/post_teardown/post-teardown.cc b/src/test/perf/post_teardown/post-teardown.cc index ca9296d71..4f42dd860 100644 --- a/src/test/perf/post_teardown/post-teardown.cc +++ b/src/test/perf/post_teardown/post-teardown.cc @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -11,7 +11,7 @@ void fill(std::vector& out, size_t count, size_t size) out.reserve(count); for (size_t i = 0; i < count; i++) { - out.push_back(snmalloc::alloc(size)); + out.push_back(snmalloc::alloc(size)); } } @@ -41,7 +41,7 @@ int main(int, char**) // Simulate the allocator already being torn down before remaining frees // (post-main / static destruction path from #809). - ThreadAlloc::teardown(); + debug_teardown(); fill(ptrs, alloc_count, obj_size); drain("Immediate dealloc after teardown", ptrs, obj_size); diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index ed068a31f..cad2620a2 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -1,25 +1,25 @@ -#include +#include #include #include #include using namespace snmalloc; -template +template void test_alloc_dealloc(size_t count, size_t size, bool write) { { MeasureTime m; m << "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) << size - << ", ZeroMem: " << std::is_same_v << ", Write: " << write; + << ", ZeroMem: " << (zero_mem == ZeroMem::YesZero) << ", Write: " << write; std::unordered_set set; // alloc 1.5x objects for (size_t i = 0; i < ((count * 3) / 2); i++) { - void* p = snmalloc::alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -41,7 +41,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) // alloc 1x objects for (size_t i = 0; i < count; i++) { - void* p = snmalloc::alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -68,18 +68,18 @@ int main(int, char**) for (size_t size = 16; size <= 128; size <<= 1) { - test_alloc_dealloc(1 << 15, size, false); - test_alloc_dealloc(1 << 15, size, true); - test_alloc_dealloc(1 << 15, size, false); - test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); } for (size_t size = 1 << 12; size <= 1 << 17; size <<= 1) { - test_alloc_dealloc(1 << 10, size, false); - test_alloc_dealloc(1 << 10, size, true); - test_alloc_dealloc(1 << 10, size, false); - test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); } return 0; From 14ad0e6356fe2c513962892d7feac2737b3caa53 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 20:25:57 +0000 Subject: [PATCH 11/24] Convert tests needing snmalloc_core.h to use testlib Convert tests that need allocator internals (sizeclasses, pagemap, etc.) to use snmalloc_core.h alongside snmalloc_testlib.h: - func/malloc (replace our_* with testlib_* prefix) - func/pagemap - func/release-rounding - func/sizeclass --- src/test/func/malloc/malloc.cc | 92 +++++++++++----------- src/test/func/pagemap/pagemap.cc | 3 +- src/test/func/release-rounding/rounding.cc | 3 +- src/test/func/sizeclass/sizeclass.cc | 3 +- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 777417ca8..3dcfb4559 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -1,12 +1,8 @@ +#include #include #include #include - -#define SNMALLOC_NAME_MANGLE(a) our_##a -#undef SNMALLOC_NO_REALLOCARRAY -#undef SNMALLOC_NO_REALLOCARR -#define SNMALLOC_BOOTSTRAP_ALLOCATOR -#include +#include using namespace snmalloc; @@ -27,8 +23,8 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) INFO("Unexpected null returned.\n"); failed = true; } - const auto alloc_size = our_malloc_usable_size(p); - auto expected_size = our_malloc_good_size(size); + const auto alloc_size = testlib_malloc_usable_size(p); + auto expected_size = testlib_malloc_good_size(size); const auto exact_size = align == 1; #ifdef __CHERI_PURE_CAPABILITY__ const auto cheri_size = __builtin_cheri_length_get(p); @@ -93,14 +89,14 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) } EXPECT(!failed, "check_result failed! {}", p); - our_free(p); + testlib_free(p); } void test_calloc(size_t nmemb, size_t size, int err, bool null) { START_TEST("calloc({}, {}) combined size {}\n", nmemb, size, nmemb * size); errno = SUCCESS; - void* p = our_calloc(nmemb, size); + void* p = testlib_calloc(nmemb, size); if (p != nullptr) { @@ -116,23 +112,23 @@ void test_realloc(void* p, size_t size, int err, bool null) { size_t old_size = 0; if (p != nullptr) - old_size = our_malloc_usable_size(p); + old_size = testlib_malloc_usable_size(p); START_TEST("realloc({}({}), {})", p, old_size, size); errno = SUCCESS; - auto new_p = our_realloc(p, size); + auto new_p = testlib_realloc(p, size); check_result(size, 1, new_p, err, null); // Realloc failure case, deallocate original block as not // handled by check_result. if (new_p == nullptr && size != 0) - our_free(p); + testlib_free(p); } void test_posix_memalign(size_t size, size_t align, int err, bool null) { START_TEST("posix_memalign(&p, {}, {})", align, size); void* p = nullptr; - errno = our_posix_memalign(&p, align, size); + errno = testlib_posix_memalign(&p, align, size); check_result(size, align, p, err, null); } @@ -140,7 +136,7 @@ void test_memalign(size_t size, size_t align, int err, bool null) { START_TEST("memalign({}, {})", align, size); errno = SUCCESS; - void* p = our_memalign(align, size); + void* p = testlib_memalign(align, size); check_result(size, align, p, err, null); } @@ -149,13 +145,13 @@ void test_reallocarray(void* p, size_t nmemb, size_t size, int err, bool null) size_t old_size = 0; size_t tsize = nmemb * size; if (p != nullptr) - old_size = our_malloc_usable_size(p); + old_size = testlib_malloc_usable_size(p); START_TEST("reallocarray({}({}), {})", p, old_size, tsize); errno = SUCCESS; - auto new_p = our_reallocarray(p, nmemb, size); + auto new_p = testlib_reallocarray(p, nmemb, size); if (new_p == nullptr && tsize != 0) - our_free(p); + testlib_free(p); check_result(tsize, 1, new_p, err, null); } @@ -165,29 +161,29 @@ void test_reallocarr( void* p = nullptr; if (size_old != (size_t)~0) - p = our_malloc(size_old); + p = testlib_malloc(size_old); START_TEST("reallocarr({}({}), {})", p, nmemb, size); errno = SUCCESS; - int r = our_reallocarr(&p, nmemb, size); + int r = testlib_reallocarr(&p, nmemb, size); EXPECT(r == err, "reallocarr failed! expected {} got {}\n", err, r); check_result(nmemb * size, 1, p, err, null); - p = our_malloc(size); + p = testlib_malloc(size); if (!p) { return; } for (size_t i = 1; i < size; i++) static_cast(p)[i] = 1; - our_reallocarr(&p, nmemb, size); + testlib_reallocarr(&p, nmemb, size); if (r != SUCCESS) - our_free(p); + testlib_free(p); for (size_t i = 1; i < size; i++) { EXPECT(static_cast(p)[i] == 1, "data consistency failed! at {}", i); } - our_free(p); + testlib_free(p); } int main(int argc, char** argv) @@ -223,11 +219,11 @@ int main(int argc, char** argv) abort(); } - our_free(nullptr); + testlib_free(nullptr); /* A very large allocation size that we expect to fail. */ const size_t too_big_size = ((size_t)-1) / 2; - check_result(too_big_size, 1, our_malloc(too_big_size), ENOMEM, true); + check_result(too_big_size, 1, testlib_malloc(too_big_size), ENOMEM, true); errno = SUCCESS; for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) @@ -235,14 +231,14 @@ int main(int argc, char** argv) const size_t size = bits::one_at_bit(sc); START_TEST("malloc: {}", size); errno = SUCCESS; - check_result(size, 1, our_malloc(size), SUCCESS, false); + check_result(size, 1, testlib_malloc(size), SUCCESS, false); errno = SUCCESS; - check_result(size + 1, 1, our_malloc(size + 1), SUCCESS, false); + check_result(size + 1, 1, testlib_malloc(size + 1), SUCCESS, false); } test_calloc(0, 0, SUCCESS, false); - our_free(nullptr); + testlib_free(nullptr); test_calloc(1, too_big_size, ENOMEM, true); errno = SUCCESS; @@ -271,35 +267,35 @@ int main(int argc, char** argv) for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); - test_realloc(our_malloc(size), size, SUCCESS, false); + test_realloc(testlib_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), too_big_size, ENOMEM, true); + test_realloc(testlib_malloc(size), too_big_size, ENOMEM, true); for (smallsizeclass_t sc2(0); sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); - test_realloc(our_malloc(size), size2, SUCCESS, false); - test_realloc(our_malloc(size + 1), size2, SUCCESS, false); + test_realloc(testlib_malloc(size), size2, SUCCESS, false); + test_realloc(testlib_malloc(size + 1), size2, SUCCESS, false); } // Check realloc(p,0), behaves like free(p), if p != nullptr - test_realloc(our_malloc(size), 0, SUCCESS, true); + test_realloc(testlib_malloc(size), 0, SUCCESS, true); } for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - test_realloc(our_malloc(size), size, SUCCESS, false); + test_realloc(testlib_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), too_big_size, ENOMEM, true); + test_realloc(testlib_malloc(size), too_big_size, ENOMEM, true); for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); INFO("size1: {}, size2:{}\n", size, size2); - test_realloc(our_malloc(size), size2, SUCCESS, false); - test_realloc(our_malloc(size + 1), size2, SUCCESS, false); + test_realloc(testlib_malloc(size), size2, SUCCESS, false); + test_realloc(testlib_malloc(size + 1), size2, SUCCESS, false); } } - test_realloc(our_malloc(64), 4194304, SUCCESS, false); + test_realloc(testlib_malloc(64), 4194304, SUCCESS, false); test_posix_memalign(0, 0, EINVAL, true); test_posix_memalign(too_big_size, 0, EINVAL, true); @@ -327,15 +323,15 @@ int main(int argc, char** argv) for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - test_reallocarray(our_malloc(size), 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, 0, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, size, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, 0, SUCCESS, false); test_reallocarray(nullptr, 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, too_big_size, ENOMEM, true); + test_reallocarray(testlib_malloc(size), 1, too_big_size, ENOMEM, true); for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); - test_reallocarray(our_malloc(size), 1, size2, SUCCESS, false); - test_reallocarray(our_malloc(size + 1), 1, size2, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, size2, SUCCESS, false); + test_reallocarray(testlib_malloc(size + 1), 1, size2, SUCCESS, false); } } @@ -348,11 +344,11 @@ int main(int argc, char** argv) test_reallocarr(size, 1, size, SUCCESS, false); test_reallocarr(size, 1, 0, SUCCESS, false); test_reallocarr(size, 2, size, SUCCESS, false); - void* p = our_malloc(size); + void* p = testlib_malloc(size); EXPECT(p != nullptr, "realloc alloc failed with {}", size); - int r = our_reallocarr(&p, 1, too_big_size); + int r = testlib_reallocarr(&p, 1, too_big_size); EXPECT(r == ENOMEM, "expected failure on allocation\n"); - our_free(p); + testlib_free(p); for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { @@ -363,7 +359,7 @@ int main(int argc, char** argv) } EXPECT( - our_malloc_usable_size(nullptr) == 0, + testlib_malloc_usable_size(nullptr) == 0, "malloc_usable_size(nullptr) should be zero"); snmalloc::debug_check_empty(); diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index a0b53689f..7a03fa1a7 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -7,8 +7,9 @@ */ #include -#include +#include #include +#include using namespace snmalloc; static constexpr size_t GRANULARITY_BITS = 20; diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 51b6e0b41..4d11eaafb 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -1,6 +1,7 @@ #include -#include +#include #include +#include using namespace snmalloc; // Check for all sizeclass that we correctly round every offset within diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 3b02003b0..ac7ec6bd8 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -1,6 +1,7 @@ #include -#include +#include #include +#include NOINLINE snmalloc::smallsizeclass_t size_to_sizeclass(size_t size) From de83961acbdb270a380ac8098142a44b08600738 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 27 Mar 2026 22:39:07 +0000 Subject: [PATCH 12/24] Move report_fatal_error/message definitions to ds_core/helpers.h Move report_fatal_error() and message() template definitions from pal/pal.h into ds_core/helpers.h. They now call error() and message_impl() which are forward-declared non-template functions in ds_core/defines.h, with implementations in pal/pal.h that delegate to DefaultPal. This breaks the dependency of ds_core/ code on pal/: any code that includes ds_core/helpers.h can call report_fatal_error() without needing PAL headers. The linker resolves error() and message_impl() from the translation unit that includes pal.h. Update testlib.h to include pal/pal_consts.h instead of pal/pal.h. Fix memory.cc to use ds_aal/ds_aal.h for address_cast. Fix protect_fork.cc to include ds_aal/prevent_fork.h explicitly. Revert low-memory.cc to snmalloc.h (needs PalNotificationObject). --- src/snmalloc/ds_core/defines.h | 19 ++++++----- src/snmalloc/ds_core/helpers.h | 33 ++++++++++++++++++ src/snmalloc/pal/pal.h | 39 ++++------------------ src/test/func/memory/memory.cc | 5 +-- src/test/func/protect_fork/protect_fork.cc | 1 + src/test/perf/low_memory/low-memory.cc | 2 +- src/test/snmalloc_testlib.h | 2 +- 7 files changed, 57 insertions(+), 44 deletions(-) diff --git a/src/snmalloc/ds_core/defines.h b/src/snmalloc/ds_core/defines.h index 8c83210ce..45cea3d70 100644 --- a/src/snmalloc/ds_core/defines.h +++ b/src/snmalloc/ds_core/defines.h @@ -123,8 +123,11 @@ namespace snmalloc static constexpr bool Debug = true; #endif - // Forwards reference so that the platform can define how to handle errors. + // Forward references so that the platform can define how to handle + // errors and messages. Definitions are provided in pal/pal.h once + // DefaultPal is known. [[noreturn]] SNMALLOC_COLD void error(const char* const str); + void message_impl(const char* const str); } // namespace snmalloc #define TOSTRING(expr) TOSTRING2(expr) @@ -213,21 +216,21 @@ namespace snmalloc namespace snmalloc { + template + SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) + {} + /** - * Forward declaration so that this can be called before the pal header is - * included. + * Forward declaration so that this can be called before helpers.h is + * included (e.g. in SNMALLOC_ASSERT macros expanded inside bits.h). */ template [[noreturn]] inline void report_fatal_error(Args... args); /** - * Forward declaration so that this can be called before the pal header is + * Forward declaration so that this can be called before helpers.h is * included. */ template inline void message(Args... args); - - template - SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) - {} } // namespace snmalloc diff --git a/src/snmalloc/ds_core/helpers.h b/src/snmalloc/ds_core/helpers.h index e5a0ca00e..7bddbfa94 100644 --- a/src/snmalloc/ds_core/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -2,6 +2,7 @@ #include "bits.h" #include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/tid.h" #include "snmalloc/stl/array.h" #include "snmalloc/stl/type_traits.h" #include "snmalloc/stl/utility.h" @@ -339,6 +340,38 @@ namespace snmalloc } }; + /** + * Report a fatal error via a PAL-specific error reporting mechanism. This + * takes a format string and a set of arguments. The format string indicates + * the remaining arguments with "{}". This could be extended later to + * support indexing fairly easily, if we ever want to localise these error + * messages. + * + * The following are supported as arguments: + * + * - Characters (`char`), printed verbatim. + * - Strings Literals (`const char*` or `const char[]`), printed verbatim. + * - Raw pointers (void*), printed as hex strings. + * - Integers (convertible to `size_t`), printed as hex strings. + * + * These types should be sufficient for allocator-related error messages. + */ + template + [[noreturn]] inline void report_fatal_error(Args... args) + { + MessageBuilder msg{stl::forward(args)...}; + error(msg.get_message()); + } + + template + inline void message(Args... args) + { + MessageBuilder msg{stl::forward(args)...}; + MessageBuilder msg_tid{ + "{}: {}", debug_get_tid(), msg.get_message()}; + message_impl(msg_tid.get_message()); + } + /** * Convenience type that has no fields / methods. */ diff --git a/src/snmalloc/pal/pal.h b/src/snmalloc/pal/pal.h index 6fbb8f347..3b3242ac5 100644 --- a/src/snmalloc/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -66,11 +66,17 @@ namespace snmalloc # error Unsupported platform #endif - [[noreturn]] SNMALLOC_SLOW_PATH inline void error(const char* const str) + [[noreturn]] SNMALLOC_SLOW_PATH SNMALLOC_USED_FUNCTION inline void + error(const char* const str) { DefaultPal::error(str); } + SNMALLOC_USED_FUNCTION inline void message_impl(const char* const str) + { + DefaultPal::message(str); + } + // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = DefaultPal::page_size; @@ -149,35 +155,4 @@ namespace snmalloc "Page size from system header does not match snmalloc config page size."); #endif - /** - * Report a fatal error via a PAL-specific error reporting mechanism. This - * takes a format string and a set of arguments. The format string indicates - * the remaining arguments with "{}". This could be extended later to - * support indexing fairly easily, if we ever want to localise these error - * messages. - * - * The following are supported as arguments: - * - * - Characters (`char`), printed verbatim. - * - Strings Literals (`const char*` or `const char[]`), printed verbatim. - * - Raw pointers (void*), printed as hex strings. - * - Integers (convertible to `size_t`), printed as hex strings. - * - * These types should be sufficient for allocator-related error messages. - */ - template - [[noreturn]] inline void report_fatal_error(Args... args) - { - MessageBuilder msg{stl::forward(args)...}; - DefaultPal::error(msg.get_message()); - } - - template - inline void message(Args... args) - { - MessageBuilder msg{stl::forward(args)...}; - MessageBuilder msg_tid{ - "{}: {}", debug_get_tid(), msg.get_message()}; - DefaultPal::message(msg_tid.get_message()); - } } // namespace snmalloc diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 7bfd4175e..607c09656 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -493,8 +494,8 @@ void test_remaining_bytes() char* p = (char*)snmalloc::alloc(size); for (size_t offset = 0; offset < size; offset++) { - auto rem = - snmalloc::remaining_bytes(address_cast(pointer_offset(p, offset))); + auto rem = snmalloc::remaining_bytes( + address_cast(pointer_offset(p, offset))); if (rem != (size - offset)) { if (rem < (size - offset) || snmalloc::is_owned(p)) diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index 4ab2d5a45..40ca10c84 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -9,6 +9,7 @@ int main() # define SNMALLOC_PTHREAD_FORK_PROTECTION # include +# include # include # include diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index ee3f68884..c9e973cc0 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h index dc51ed67a..32cf2c2ea 100644 --- a/src/test/snmalloc_testlib.h +++ b/src/test/snmalloc_testlib.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include From 2f65c2c07640cb566565fab872e4b4b340d8d0b2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 28 Mar 2026 16:14:43 +0000 Subject: [PATCH 13/24] Remove unnecessary errno.h include from snmalloc_testlib.h The header doesn't use errno itself. Tests that need it can include it directly. --- src/test/snmalloc_testlib.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h index 32cf2c2ea..1cf1fb162 100644 --- a/src/test/snmalloc_testlib.h +++ b/src/test/snmalloc_testlib.h @@ -13,7 +13,6 @@ * (distinct from the templates, no ODR conflict) defined in testlib.cc. */ -#include #include #include #include From 01d650ee35b6c7bd4b3627da9a3fd3f2beb57abc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 28 Mar 2026 18:32:42 +0000 Subject: [PATCH 14/24] Convert protect_fork, multi_atexit, multi_threadatexit Use minimal includes for protect_fork (ds_core/helpers.h + ds_aal/prevent_fork.h). Fix missed testlib conversions for multi_atexit and multi_threadatexit. --- src/test/func/multi_atexit/multi_atexit.cc | 2 +- src/test/func/multi_threadatexit/multi_threadatexit.cc | 2 +- src/test/func/protect_fork/protect_fork.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/func/multi_atexit/multi_atexit.cc b/src/test/func/multi_atexit/multi_atexit.cc index e4f5ed327..d2e3978b7 100644 --- a/src/test/func/multi_atexit/multi_atexit.cc +++ b/src/test/func/multi_atexit/multi_atexit.cc @@ -13,7 +13,7 @@ #endif #ifdef RUN_TEST -# include +# include # include void do_nothing() {} diff --git a/src/test/func/multi_threadatexit/multi_threadatexit.cc b/src/test/func/multi_threadatexit/multi_threadatexit.cc index 85f918f93..10322b1a6 100644 --- a/src/test/func/multi_threadatexit/multi_threadatexit.cc +++ b/src/test/func/multi_threadatexit/multi_threadatexit.cc @@ -15,7 +15,7 @@ #endif #ifdef RUN_TEST -# include +# include # include template diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index 40ca10c84..52655582f 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -9,8 +9,8 @@ int main() # define SNMALLOC_PTHREAD_FORK_PROTECTION # include +# include # include -# include # include void simulate_allocation() From b1b1c133ba46b220e0bd83cf57352617ff5244ed Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 28 Mar 2026 21:23:33 +0000 Subject: [PATCH 15/24] Split allocconfig.h: extract sizeclassconfig.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create ds_core/sizeclassconfig.h with the mitigation-independent sizeclass constants (INTERMEDIATE_BITS, MIN_ALLOC_STEP_SIZE, etc.). This file depends only on bits.h — no mitigations.h. Update sizeclassstatic.h to include sizeclassconfig.h instead of allocconfig.h. allocconfig.h now includes sizeclassconfig.h and keeps only the mitigation-dependent constants (MIN_OBJECT_COUNT, DEALLOC_BATCH_*, MAX_SLAB_SPAN_*, etc.). This makes the testlib header chain fully mitigation-independent: testlib.h -> sizeclassstatic.h -> sizeclassconfig.h -> bits.h --- src/snmalloc/ds_core/allocconfig.h | 103 +------------------------ src/snmalloc/ds_core/ds_core.h | 3 +- src/snmalloc/ds_core/sizeclassconfig.h | 85 ++++++++++++++++++++ src/snmalloc/ds_core/sizeclassstatic.h | 3 +- 4 files changed, 88 insertions(+), 106 deletions(-) create mode 100644 src/snmalloc/ds_core/sizeclassconfig.h diff --git a/src/snmalloc/ds_core/allocconfig.h b/src/snmalloc/ds_core/allocconfig.h index 5ee00af76..aa4cde73a 100644 --- a/src/snmalloc/ds_core/allocconfig.h +++ b/src/snmalloc/ds_core/allocconfig.h @@ -1,101 +1,14 @@ #pragma once -#include "bits.h" #include "mitigations.h" +#include "sizeclassconfig.h" namespace snmalloc { - // 0 intermediate bits results in power of 2 small allocs. 1 intermediate - // bit gives additional sizeclasses at the midpoint between each power of 2. - // 2 intermediate bits gives 3 intermediate sizeclasses, etc. - static constexpr size_t INTERMEDIATE_BITS = -#ifdef USE_INTERMEDIATE_BITS - USE_INTERMEDIATE_BITS -#else - 2 -#endif - ; - - // The remaining values are derived, not configurable. - static constexpr size_t POINTER_BITS = - bits::next_pow2_bits_const(sizeof(uintptr_t)); - - // Used to isolate values on cache lines to prevent false sharing. - static constexpr size_t CACHELINE_SIZE = 64; - - /// The "machine epsilon" for the small sizeclass machinery. - static constexpr size_t MIN_ALLOC_STEP_SIZE = -#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE) - SNMALLOC_MIN_ALLOC_STEP_SIZE; -#else - 2 * sizeof(void*); -#endif - - /// Derived from MIN_ALLOC_STEP_SIZE - static constexpr size_t MIN_ALLOC_STEP_BITS = - bits::ctz_const(MIN_ALLOC_STEP_SIZE); - static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE)); - - /** - * Minimum allocation size is space for two pointers. If the small sizeclass - * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is - * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must - * be large or not a power of two, those smaller size classes will be unused. - */ - static constexpr size_t MIN_ALLOC_SIZE = -#if defined(SNMALLOC_MIN_ALLOC_SIZE) - SNMALLOC_MIN_ALLOC_SIZE; -#else - 2 * sizeof(void*); -#endif - - // Minimum slab size. -#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) - /* - * QEMU user-mode, up through and including v7.2.0-rc4, the latest tag at the - * time of this writing, does not use a tree of any sort to store its opinion - * of the address space, allocating an amount of memory linear in the size of - * any created map, not the number of pages actually used. This is - * exacerbated in and after qemu v6 (or, more specifically, d9c58585), which - * grew the proportionality constant. - * - * In any case, for our CI jobs, then, use a larger minimum chunk size (that - * is, pagemap granularity) than by default to reduce the size of the - * pagemap. We can't raise this *too* much, lest we hit constexpr step - * limits in the sizeclasstable magic! 17 bits seems to be the sweet spot - * and means that any of our tests can run in a little under 2 GiB of RSS - * even on QEMU versions after v6. - */ - static constexpr size_t MIN_CHUNK_BITS = static_cast(17); -#else - static constexpr size_t MIN_CHUNK_BITS = static_cast(14); -#endif - static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); - // Minimum number of objects on a slab static constexpr size_t MIN_OBJECT_COUNT = mitigations(random_larger_thresholds) ? 13 : 4; - // Maximum size of an object that uses sizeclasses. -#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) - /* - * As a consequence of our significantly larger minimum chunk size, we need - * to raise the threshold for what constitutes a large object (which must - * be a multiple of the minimum chunk size). Extend the space of small - * objects up enough to match yet preserve the notion that there exist small - * objects larger than MIN_CHUNK_SIZE. - */ - static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 19; -#else - static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 16; -#endif - static constexpr size_t MAX_SMALL_SIZECLASS_SIZE = - bits::one_at_bit(MAX_SMALL_SIZECLASS_BITS); - - static_assert( - MAX_SMALL_SIZECLASS_SIZE >= MIN_CHUNK_SIZE, - "Large sizes need to be representable by as a multiple of MIN_CHUNK_SIZE"); - /** * The number of bits needed to count the number of objects within a slab. * @@ -162,20 +75,6 @@ namespace snmalloc static constexpr size_t DEALLOC_BATCH_RINGS = DEALLOC_BATCH_RING_ASSOC * bits::one_at_bit(DEALLOC_BATCH_RING_SET_BITS); - static_assert( - INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS, - "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); - static_assert( - MIN_ALLOC_SIZE >= (sizeof(void*) * 2), - "MIN_ALLOC_SIZE must be sufficient for two pointers"); - static_assert( - 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >= - bits::next_pow2_const(MIN_ALLOC_SIZE), - "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE"); - static_assert( - MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE, - "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE"); - // Return remote small allocs when the local cache reaches this size. static constexpr int64_t REMOTE_CACHE = #ifdef USE_REMOTE_CACHE diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index 0eec777be..25cf7daaa 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -16,7 +16,6 @@ #include "mitigations.h" #include "ptrwrap.h" #include "redblacktree.h" +#include "sizeclassconfig.h" #include "sizeclassstatic.h" -#include "tid.h" - #include "tid.h" \ No newline at end of file diff --git a/src/snmalloc/ds_core/sizeclassconfig.h b/src/snmalloc/ds_core/sizeclassconfig.h new file mode 100644 index 000000000..05224f81f --- /dev/null +++ b/src/snmalloc/ds_core/sizeclassconfig.h @@ -0,0 +1,85 @@ +#pragma once + +#include "bits.h" + +namespace snmalloc +{ + // 0 intermediate bits results in power of 2 small allocs. 1 intermediate + // bit gives additional sizeclasses at the midpoint between each power of 2. + // 2 intermediate bits gives 3 intermediate sizeclasses, etc. + static constexpr size_t INTERMEDIATE_BITS = +#ifdef USE_INTERMEDIATE_BITS + USE_INTERMEDIATE_BITS +#else + 2 +#endif + ; + + // The remaining values are derived, not configurable. + static constexpr size_t POINTER_BITS = + bits::next_pow2_bits_const(sizeof(uintptr_t)); + + // Used to isolate values on cache lines to prevent false sharing. + static constexpr size_t CACHELINE_SIZE = 64; + + /// The "machine epsilon" for the small sizeclass machinery. + static constexpr size_t MIN_ALLOC_STEP_SIZE = +#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE) + SNMALLOC_MIN_ALLOC_STEP_SIZE; +#else + 2 * sizeof(void*); +#endif + + /// Derived from MIN_ALLOC_STEP_SIZE + static constexpr size_t MIN_ALLOC_STEP_BITS = + bits::ctz_const(MIN_ALLOC_STEP_SIZE); + static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE)); + + /** + * Minimum allocation size is space for two pointers. If the small sizeclass + * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is + * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must + * be large or not a power of two, those smaller size classes will be unused. + */ + static constexpr size_t MIN_ALLOC_SIZE = +#if defined(SNMALLOC_MIN_ALLOC_SIZE) + SNMALLOC_MIN_ALLOC_SIZE; +#else + 2 * sizeof(void*); +#endif + + // Minimum slab size. +#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) + static constexpr size_t MIN_CHUNK_BITS = static_cast(17); +#else + static constexpr size_t MIN_CHUNK_BITS = static_cast(14); +#endif + static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); + + // Maximum size of an object that uses sizeclasses. +#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) + static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 19; +#else + static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 16; +#endif + static constexpr size_t MAX_SMALL_SIZECLASS_SIZE = + bits::one_at_bit(MAX_SMALL_SIZECLASS_BITS); + + static_assert( + MAX_SMALL_SIZECLASS_SIZE >= MIN_CHUNK_SIZE, + "Large sizes need to be representable by as a multiple of MIN_CHUNK_SIZE"); + + static_assert( + INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS, + "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); + static_assert( + MIN_ALLOC_SIZE >= (sizeof(void*) * 2), + "MIN_ALLOC_SIZE must be sufficient for two pointers"); + static_assert( + 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >= + bits::next_pow2_const(MIN_ALLOC_SIZE), + "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE"); + static_assert( + MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE, + "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE"); +} // namespace snmalloc diff --git a/src/snmalloc/ds_core/sizeclassstatic.h b/src/snmalloc/ds_core/sizeclassstatic.h index eba2181d4..95cacf01b 100644 --- a/src/snmalloc/ds_core/sizeclassstatic.h +++ b/src/snmalloc/ds_core/sizeclassstatic.h @@ -1,7 +1,6 @@ #pragma once -#include "allocconfig.h" -#include "bits.h" +#include "sizeclassconfig.h" namespace snmalloc { From f3448e6e837422d13c9e3cda5e1dabbccabbd97e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 28 Mar 2026 21:57:11 +0000 Subject: [PATCH 16/24] Create mitigations/ layer between ds_aal/ and pal/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move mitigations.h, allocconfig.h, and cheri.h from ds_core/ to a new mitigations/ directory. These files depend on SNMALLOC_CHECK_CLIENT compile-time flags and don't belong in ds_core/ (which should be mitigation-independent). New include hierarchy: ds_core/ → aal/ → ds_aal/ → mitigations/ → pal/ → ds/ → mem/ ds_core/ is now fully mitigation-independent. The testlib header (which includes only ds_core/ headers) produces identical output regardless of SNMALLOC_CHECK_CLIENT, enabling tests to be compiled once and linked against both fast and check testlib variants. --- src/snmalloc/ds_core/ds_core.h | 3 --- src/snmalloc/mem/secondary/default.h | 2 +- src/snmalloc/{ds_core => mitigations}/allocconfig.h | 2 +- src/snmalloc/{ds_core => mitigations}/cheri.h | 0 src/snmalloc/{ds_core => mitigations}/mitigations.h | 2 +- src/snmalloc/mitigations/mitigations_all.h | 12 ++++++++++++ src/snmalloc/pal/pal.h | 6 ++++-- 7 files changed, 19 insertions(+), 8 deletions(-) rename src/snmalloc/{ds_core => mitigations}/allocconfig.h (98%) rename src/snmalloc/{ds_core => mitigations}/cheri.h (100%) rename src/snmalloc/{ds_core => mitigations}/mitigations.h (99%) create mode 100644 src/snmalloc/mitigations/mitigations_all.h diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index 25cf7daaa..0c22a35bd 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -7,13 +7,10 @@ * snmalloc. */ -#include "allocconfig.h" #include "bits.h" -#include "cheri.h" #include "concept.h" #include "defines.h" #include "helpers.h" -#include "mitigations.h" #include "ptrwrap.h" #include "redblacktree.h" #include "sizeclassconfig.h" diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index ae5d53bfb..1fc86af46 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -1,7 +1,7 @@ #pragma once #include "snmalloc/ds_core/defines.h" -#include "snmalloc/ds_core/mitigations.h" +#include "snmalloc/mitigations/mitigations.h" #include diff --git a/src/snmalloc/ds_core/allocconfig.h b/src/snmalloc/mitigations/allocconfig.h similarity index 98% rename from src/snmalloc/ds_core/allocconfig.h rename to src/snmalloc/mitigations/allocconfig.h index aa4cde73a..fcc8217d1 100644 --- a/src/snmalloc/ds_core/allocconfig.h +++ b/src/snmalloc/mitigations/allocconfig.h @@ -1,7 +1,7 @@ #pragma once #include "mitigations.h" -#include "sizeclassconfig.h" +#include "../ds_core/sizeclassconfig.h" namespace snmalloc { diff --git a/src/snmalloc/ds_core/cheri.h b/src/snmalloc/mitigations/cheri.h similarity index 100% rename from src/snmalloc/ds_core/cheri.h rename to src/snmalloc/mitigations/cheri.h diff --git a/src/snmalloc/ds_core/mitigations.h b/src/snmalloc/mitigations/mitigations.h similarity index 99% rename from src/snmalloc/ds_core/mitigations.h rename to src/snmalloc/mitigations/mitigations.h index 4f5200853..5cd46911f 100644 --- a/src/snmalloc/ds_core/mitigations.h +++ b/src/snmalloc/mitigations/mitigations.h @@ -1,5 +1,5 @@ #pragma once -#include "defines.h" +#include "../ds_core/defines.h" #include diff --git a/src/snmalloc/mitigations/mitigations_all.h b/src/snmalloc/mitigations/mitigations_all.h new file mode 100644 index 000000000..379455642 --- /dev/null +++ b/src/snmalloc/mitigations/mitigations_all.h @@ -0,0 +1,12 @@ +#pragma once +/** + * The mitigations layer provides compile-time configuration for security + * mitigations. It sits between ds_aal/ and pal/ in the include hierarchy. + * + * Files in this directory may include ds_core/ and ds_aal/ but not pal/ + * or anything above. + */ + +#include "allocconfig.h" +#include "cheri.h" +#include "mitigations.h" diff --git a/src/snmalloc/pal/pal.h b/src/snmalloc/pal/pal.h index 3b3242ac5..884775459 100644 --- a/src/snmalloc/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -7,12 +7,14 @@ * may hide some of this or provide its own abstractions for sandboxing or * isolating heaps. * - * Files in this directory may depend on the architecture abstraction and core - * layers (`aal` and `ds_core`, respectively) but nothing else in snmalloc. + * Files in this directory may depend on the architecture abstraction, core, + * and mitigations layers (`aal`, `ds_core`, and `mitigations/` respectively) + * but nothing else in snmalloc. */ #pragma once #include "../aal/aal.h" +#include "../mitigations/mitigations_all.h" #include "pal_concept.h" #include "pal_consts.h" From 2dba9afb10151124a4f7e2dcdd877020de391c43 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 28 Mar 2026 22:02:19 +0000 Subject: [PATCH 17/24] Compile testlib-only tests once for both fast and check Testlib-only tests are mitigation-independent: compile into shared OBJECT libraries. Also fix MSVC C4701 warning in cleanup_unused and add unistd.h to protect_fork. --- CMakeLists.txt | 43 +++++++++++++++++++++- src/snmalloc/global/globalalloc.h | 20 +++++----- src/test/func/protect_fork/protect_fork.cc | 1 + 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fea1b75d..1515fb44e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,16 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(TEST_CLEANUP ${SNMALLOC_CLEANUP}) endif() + # Tests that only include snmalloc_testlib.h (no snmalloc.h or snmalloc_core.h). + # These are mitigation-independent and can be compiled once, then linked + # against both fast and check testlib variants. + set(TESTLIB_ONLY_TESTS + bits first_operation memory_usage multi_atexit multi_threadatexit + redblack statistics teardown + contention external_pointer large_alloc lotsofthreads post_teardown + singlethread startup + ) + function(make_tests TAG DEFINES) foreach(TEST_CATEGORY ${TEST_CATEGORIES}) message(VERBOSE "Adding ${TAG}/${TEST_CATEGORY} tests") @@ -488,7 +498,32 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC) set(TESTNAME "${TEST_CATEGORY}-${TEST}-${TAG}") - add_executable(${TESTNAME} ${SRC}) + # Check if this test is testlib-only (mitigation-independent). + # If so, compile the source once into an object library and reuse + # across flavours. + list(FIND TESTLIB_ONLY_TESTS ${TEST} _testlib_only_idx) + set(OBJLIBNAME "${TEST_CATEGORY}-${TEST}-obj") + if (NOT ${_testlib_only_idx} EQUAL -1 AND NOT TARGET ${OBJLIBNAME}) + # First flavour to see this test creates the object library. + add_library(${OBJLIBNAME} OBJECT ${SRC}) + target_link_libraries(${OBJLIBNAME} snmalloc) + target_compile_definitions(${OBJLIBNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") + add_warning_flags(${OBJLIBNAME}) + if(SNMALLOC_SANITIZER) + target_compile_options(${OBJLIBNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) + if (${SNMALLOC_SANITIZER} MATCHES "thread") + target_compile_definitions(${OBJLIBNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED) + endif() + endif() + endif() + + if (NOT ${_testlib_only_idx} EQUAL -1) + # Testlib-only: link the shared object library against this + # flavour's testlib. + add_executable(${TESTNAME} $) + else() + add_executable(${TESTNAME} ${SRC}) + endif() if(SNMALLOC_SANITIZER) target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) @@ -551,6 +586,12 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) if (NOT DEFINES STREQUAL " ") target_compile_definitions(${LIBNAME} PRIVATE ${DEFINES}) endif() + if(SNMALLOC_SANITIZER) + target_compile_options(${LIBNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) + if (${SNMALLOC_SANITIZER} MATCHES "thread") + target_compile_definitions(${LIBNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED) + endif() + endif() add_warning_flags(${LIBNAME}) endfunction() diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 7499feea5..5a80a26f3 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -17,19 +17,19 @@ namespace snmalloc // Handling the message queue for each stack is non-atomic. auto* first = AllocPool::extract(); auto* alloc = first; - decltype(alloc) last; - if (alloc != nullptr) - { - while (alloc != nullptr) - { - alloc->flush(); - last = alloc; - alloc = AllocPool::extract(alloc); - } + if (alloc == nullptr) + return; - AllocPool::restore(first, last); + decltype(alloc) last = alloc; + while (alloc != nullptr) + { + alloc->flush(); + last = alloc; + alloc = AllocPool::extract(alloc); } + + AllocPool::restore(first, last); } /** diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index 52655582f..f008f7aca 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -12,6 +12,7 @@ int main() # include # include # include +# include void simulate_allocation() { From 1a66f10626102a68146efac792fe7d5dc0a60810 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sun, 29 Mar 2026 09:48:20 +0100 Subject: [PATCH 18/24] Document mitigations/ layer in include hierarchy README --- src/snmalloc/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/snmalloc/README.md b/src/snmalloc/README.md index 9f593ac1d..2549320fb 100644 --- a/src/snmalloc/README.md +++ b/src/snmalloc/README.md @@ -10,6 +10,9 @@ These are arranged in a hierarchy such that each of the directories may include This layer provides abstractions over CPU-specific intrinsics and defines things such as the virtual address-space size. There is a single AAL for an snmalloc instantiation. - `ds_aal/` provides data structures that depend on the AAL. + - `mitigations/` provides compile-time configuration for security mitigations. + This includes the `mitigations()` function (controlled by `SNMALLOC_CHECK_CLIENT`), mitigation-dependent allocator constants, and CHERI capability checks. + Layers below this (`ds_core/`, `aal/`, `ds_aal/`) are mitigation-independent, which allows code that only includes those layers to be compiled once regardless of mitigation settings. - `pal/` provides the platform abstraction layer (PAL). This exposes OS- or environment-specific abstractions into the rest of the code. An snmalloc instantiation may use more than one PAL, including ones provided by the user. From c4f359d3b190c3b014aa610bdc42b7dc2d4eabf2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sun, 29 Mar 2026 09:51:22 +0100 Subject: [PATCH 19/24] Use SNMALLOC_API macro for non-template API functions Introduce SNMALLOC_API macro for public API functions (dealloc, debug_teardown, libc::malloc, etc.) that need standalone definitions when compiled into a static library. Defaults to SNMALLOC_FAST_PATH_INLINE (normal header-only usage). testlib.cc defines SNMALLOC_API as NOINLINE on MSVC before including snmalloc.h, producing strong definitions that ARM64EC exit-thunks can reference. On GCC/Clang, __attribute__((used)) via the existing SNMALLOC_USED_FUNCTION annotation suffices. --- BUILD.bazel | 2 ++ CMakeLists.txt | 1 + src/snmalloc/global/globalalloc.h | 16 ++++++++--- src/snmalloc/global/libc.h | 34 ++++++++++++------------ src/test/snmalloc_testlib.cc | 44 +++++++------------------------ 5 files changed, 41 insertions(+), 56 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 0d734ad4f..70af3d5f3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,6 +6,7 @@ filegroup( [ "src/snmalloc/**/*", "src/test/*.h", + "src/test/*.cc", "CMakeLists.txt", ], ), @@ -39,6 +40,7 @@ CMAKE_FLAGS = { "SNMALLOC_USE_SELF_VENDORED_STL": "OFF", "SNMALLOC_IPO": "ON", "USE_SNMALLOC_STATS": "ON", + "SNMALLOC_BUILD_TESTING": "OFF", } | select({ ":release_with_debug": {"CMAKE_BUILD_TYPE": "RelWithDebInfo"}, ":release": {"CMAKE_BUILD_TYPE": "Release"}, diff --git a/CMakeLists.txt b/CMakeLists.txt index 1515fb44e..1be1f2afd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -801,6 +801,7 @@ install(DIRECTORY src/snmalloc/ds_aal DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/ds_core DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/global DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/mem DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/mitigations DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/override DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/pal DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/stl DESTINATION include/snmalloc) diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 5a80a26f3..7ca9c06be 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -3,6 +3,14 @@ #include "../mem/mem.h" #include "threadalloc.h" +// Non-template API functions use SNMALLOC_API for their linkage specifier. +// Normally inline; can be overridden to NOINLINE (via SNMALLOC_API_NOINLINE) +// to produce strong definitions in a standalone compilation unit. +#ifndef SNMALLOC_API +# define SNMALLOC_API SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE +# define SNMALLOC_API_SLOW SNMALLOC_USED_FUNCTION inline +#endif + namespace snmalloc { template @@ -363,12 +371,12 @@ namespace snmalloc aligned_size(align, size)); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) + SNMALLOC_API void dealloc(void* p) { ThreadAlloc::get().dealloc(p); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + SNMALLOC_API void dealloc(void* p, size_t size) { check_size(p, size); @@ -382,7 +390,7 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + SNMALLOC_API void dealloc(void* p, size_t size, size_t align) { auto rsize = aligned_size(align, size); @@ -390,7 +398,7 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void debug_teardown() + SNMALLOC_API void debug_teardown() { return ThreadAlloc::teardown(); } diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index c1c389f2e..16f2c9387 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -19,22 +19,22 @@ namespace snmalloc::libc return err; } - SNMALLOC_USED_FUNCTION inline void* __malloc_end_pointer(void* ptr) + SNMALLOC_API_SLOW void* __malloc_end_pointer(void* ptr) { return snmalloc::external_pointer(ptr); } - SNMALLOC_USED_FUNCTION inline void* __malloc_start_pointer(void* ptr) + SNMALLOC_API_SLOW void* __malloc_start_pointer(void* ptr) { return snmalloc::external_pointer(ptr); } - SNMALLOC_USED_FUNCTION inline void* __malloc_last_byte_pointer(void* ptr) + SNMALLOC_API_SLOW void* __malloc_last_byte_pointer(void* ptr) { return snmalloc::external_pointer(ptr); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) + SNMALLOC_API void* malloc(size_t size) { return snmalloc::alloc(size); } @@ -43,7 +43,7 @@ namespace snmalloc::libc * Allocate for a pre-computed small sizeclass. * Use is_small_sizeclass() + size_to_sizeclass_const() to get the class. */ - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + SNMALLOC_API void* malloc_small(smallsizeclass_t sizeclass) { return snmalloc::alloc(sizeclass); @@ -52,30 +52,30 @@ namespace snmalloc::libc /** * Allocate zeroed memory for a pre-computed small sizeclass. */ - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + SNMALLOC_API void* malloc_small_zero(smallsizeclass_t sizeclass) { return snmalloc::alloc(sizeclass); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void free(void* ptr) + SNMALLOC_API void free(void* ptr) { dealloc(ptr); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + SNMALLOC_API void free_sized(void* ptr, size_t size) { dealloc(ptr, size); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + SNMALLOC_API void free_aligned_sized(void* ptr, size_t alignment, size_t size) { dealloc(ptr, size, alignment); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + SNMALLOC_API void* calloc(size_t nmemb, size_t size) { bool overflow = false; @@ -87,7 +87,7 @@ namespace snmalloc::libc return alloc(sz); } - SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + SNMALLOC_API void* realloc(void* ptr, size_t size) { // Glibc treats @@ -137,12 +137,12 @@ namespace snmalloc::libc return p; } - SNMALLOC_USED_FUNCTION inline size_t malloc_usable_size(const void* ptr) + SNMALLOC_API_SLOW size_t malloc_usable_size(const void* ptr) { return alloc_size(ptr); } - SNMALLOC_USED_FUNCTION inline void* + SNMALLOC_API_SLOW void* reallocarray(void* ptr, size_t nmemb, size_t size) { bool overflow = false; @@ -154,7 +154,7 @@ namespace snmalloc::libc return realloc(ptr, sz); } - SNMALLOC_USED_FUNCTION inline int + SNMALLOC_API_SLOW int reallocarr(void* ptr_, size_t nmemb, size_t size) { int err = errno; @@ -190,7 +190,7 @@ namespace snmalloc::libc return 0; } - SNMALLOC_USED_FUNCTION inline void* + SNMALLOC_API_SLOW void* memalign(size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY(alignment == 0 || !bits::is_pow2(alignment))) @@ -201,13 +201,13 @@ namespace snmalloc::libc return alloc_aligned(alignment, size); } - SNMALLOC_USED_FUNCTION inline void* + SNMALLOC_API_SLOW void* aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } - SNMALLOC_USED_FUNCTION inline int + SNMALLOC_API_SLOW int posix_memalign(void** memptr, size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY( diff --git a/src/test/snmalloc_testlib.cc b/src/test/snmalloc_testlib.cc index d5f8e31df..57b58f8ed 100644 --- a/src/test/snmalloc_testlib.cc +++ b/src/test/snmalloc_testlib.cc @@ -22,6 +22,15 @@ * ODR-conflicting with the templates). */ +// On MSVC (where SNMALLOC_USED_FUNCTION is empty), override the testlib +// inline macros to produce strong (non-inline) definitions so the linker +// can resolve ARM64EC exit-thunks. On GCC/Clang, __attribute__((used)) +// already handles this. +#ifdef _MSC_VER +# define SNMALLOC_API NOINLINE +# define SNMALLOC_API_SLOW NOINLINE +#endif + #include namespace snmalloc @@ -167,41 +176,6 @@ namespace snmalloc { Aal::pause(); } - - // -- Force emission of inline functions for MSVC (where USED_FUNCTION is - // empty). Taking the address of each inline function forces the compiler to - // emit a definition in this TU, making it available to testlib consumers. - - SNMALLOC_USED_FUNCTION static const volatile void* force_emit_global[] = { - reinterpret_cast( - static_cast(&dealloc)), - reinterpret_cast( - static_cast(&dealloc)), - reinterpret_cast( - static_cast(&dealloc)), - reinterpret_cast(&debug_teardown), - }; - - namespace libc - { - SNMALLOC_USED_FUNCTION static const volatile void* force_emit_libc[] = { - reinterpret_cast(&__malloc_start_pointer), - reinterpret_cast(&__malloc_last_byte_pointer), - reinterpret_cast(&__malloc_end_pointer), - reinterpret_cast(&malloc), - reinterpret_cast(&free), - reinterpret_cast( - static_cast(&calloc)), - reinterpret_cast( - static_cast(&realloc)), - reinterpret_cast(&malloc_usable_size), - reinterpret_cast(&memalign), - reinterpret_cast(&aligned_alloc), - reinterpret_cast(&posix_memalign), - reinterpret_cast(&malloc_small), - reinterpret_cast(&malloc_small_zero), - }; - } // namespace libc } // namespace snmalloc // -- override/malloc.cc symbols with testlib_ prefix ----------------------- From f26d146dbffc26d68f136a082c3958b985769a71 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 30 Mar 2026 17:09:43 +0100 Subject: [PATCH 20/24] Change remaining_bytes API to take const void* instead of address_t Add a const void* overload of remaining_bytes alongside the existing address_t version. The testlib API uses const void* so callers don't need address_cast or ds_aal headers. Remove address_t from testlib.h. This makes memory.cc a testlib-only test (no ds_aal.h needed), adding it to the compile-once list. --- CMakeLists.txt | 2 +- src/snmalloc/global/globalalloc.h | 6 ++++++ src/test/func/memory/memory.cc | 4 +--- src/test/snmalloc_testlib.cc | 2 +- src/test/snmalloc_testlib.h | 6 +----- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1be1f2afd..f1c7e7153 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -483,7 +483,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) # These are mitigation-independent and can be compiled once, then linked # against both fast and check testlib variants. set(TESTLIB_ONLY_TESTS - bits first_operation memory_usage multi_atexit multi_threadatexit + bits first_operation memory memory_usage multi_atexit multi_threadatexit redblack statistics teardown contention external_pointer large_alloc lotsofthreads post_teardown singlethread startup diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 7ca9c06be..748002b00 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -151,6 +151,12 @@ namespace snmalloc return snmalloc::remaining_bytes(sizeclass, p); } + template + size_t SNMALLOC_FAST_PATH_INLINE remaining_bytes(const void* p) + { + return remaining_bytes(address_cast(p)); + } + /** * Returns the byte offset into an object. * diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 607c09656..08e101a15 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -494,8 +493,7 @@ void test_remaining_bytes() char* p = (char*)snmalloc::alloc(size); for (size_t offset = 0; offset < size; offset++) { - auto rem = snmalloc::remaining_bytes( - address_cast(pointer_offset(p, offset))); + auto rem = snmalloc::remaining_bytes(pointer_offset(p, offset)); if (rem != (size - offset)) { if (rem < (size - offset) || snmalloc::is_owned(p)) diff --git a/src/test/snmalloc_testlib.cc b/src/test/snmalloc_testlib.cc index 57b58f8ed..9face3034 100644 --- a/src/test/snmalloc_testlib.cc +++ b/src/test/snmalloc_testlib.cc @@ -58,7 +58,7 @@ namespace snmalloc return alloc_size(p); } - size_t remaining_bytes(address_t p) + size_t remaining_bytes(const void* p) { return remaining_bytes(p); } diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h index 1cf1fb162..aba8808ab 100644 --- a/src/test/snmalloc_testlib.h +++ b/src/test/snmalloc_testlib.h @@ -22,10 +22,6 @@ namespace snmalloc { - // Forward declarations sufficient for the API surface. - // address_t: uintptr_t on all non-CHERI platforms. - using address_t = uintptr_t; - template class DefaultConts; // Uninit / Zero are usable as template arguments even without the full @@ -55,7 +51,7 @@ namespace snmalloc // -- Non-template wrappers for Config-templated functions ---------------- size_t alloc_size(const void* p); - size_t remaining_bytes(address_t p); + size_t remaining_bytes(const void* p); bool is_owned(void* p); void debug_check_empty(bool* result = nullptr); void debug_in_use(size_t count); From 1e88f8c53ab5a4850eb52c8f15b34e12b581b8c7 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 30 Mar 2026 17:21:49 +0100 Subject: [PATCH 21/24] Add libc::memcpy with Checked/ReadsChecked template parameters Expose snmalloc's checked memcpy through the libc API surface: libc::memcpy(dst, src, len) Checked controls destination bounds checking, ReadsChecked controls source bounds checking (defaults to same as Checked). Explicit instantiations provided in testlib for the common variants: , , . --- src/snmalloc/global/libc.h | 13 +++++++++++++ src/test/snmalloc_testlib.cc | 8 ++++++++ src/test/snmalloc_testlib.h | 3 +++ 3 files changed, 24 insertions(+) diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index 16f2c9387..f80fa9d1d 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -1,6 +1,7 @@ #pragma once #include "globalalloc.h" +#include "memcpy.h" #include #include @@ -225,4 +226,16 @@ namespace snmalloc::libc *memptr = p; return 0; } + + /** + * Checked memcpy through the libc API surface. + * @tparam Checked check the destination is within a snmalloc allocation + * @tparam ReadsChecked also check the source + */ + template + SNMALLOC_API void* + memcpy(void* dst, const void* src, size_t len) + { + return snmalloc::memcpy(dst, src, len); + } } // namespace snmalloc::libc diff --git a/src/test/snmalloc_testlib.cc b/src/test/snmalloc_testlib.cc index 9face3034..b9dce251e 100644 --- a/src/test/snmalloc_testlib.cc +++ b/src/test/snmalloc_testlib.cc @@ -160,6 +160,14 @@ namespace snmalloc return MAX_SMALL_SIZECLASS_BITS; } + // -- Explicit instantiations of libc::memcpy variants -------------------- + namespace libc + { + template void* memcpy(void*, const void*, size_t); + template void* memcpy(void*, const void*, size_t); + template void* memcpy(void*, const void*, size_t); + } // namespace libc + // -- PAL/AAL wrappers ---------------------------------------------------- size_t pal_address_bits() diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h index aba8808ab..343362999 100644 --- a/src/test/snmalloc_testlib.h +++ b/src/test/snmalloc_testlib.h @@ -125,6 +125,9 @@ namespace snmalloc void* __malloc_start_pointer(void* ptr); void* __malloc_last_byte_pointer(void* ptr); void* __malloc_end_pointer(void* ptr); + + template + void* memcpy(void* dst, const void* src, size_t len); } // namespace libc /** From 89b236173843a6834deb7b53061e59d6365541d8 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 30 Mar 2026 17:51:08 +0100 Subject: [PATCH 22/24] Remove DefaultPal dependency from test/helpers.h Replace DefaultPal::message() calls in START_TEST and INFO macros with snmalloc::message() which is now defined in ds_core/helpers.h. This removes the PAL dependency from test infrastructure, allowing tests that only need ds_core/ or ds_aal/ headers to avoid including pal/pal.h. --- src/test/helpers.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/helpers.h b/src/test/helpers.h index 30f6e4655..d8738ddc9 100644 --- a/src/test/helpers.h +++ b/src/test/helpers.h @@ -18,8 +18,7 @@ namespace snmalloc do \ { \ current_test = __PRETTY_FUNCTION__; \ - MessageBuilder<1024> mb{"Starting test: " msg "\n", ##__VA_ARGS__}; \ - DefaultPal::message(mb.get_message()); \ + snmalloc::message<1024>("Starting test: " msg, ##__VA_ARGS__); \ } while (0) /** @@ -32,8 +31,7 @@ namespace snmalloc #define INFO(msg, ...) \ do \ { \ - MessageBuilder<1024> mb{msg "\n", ##__VA_ARGS__}; \ - DefaultPal::message(mb.get_message()); \ + snmalloc::message<1024>(msg, ##__VA_ARGS__); \ } while (0) } From 6e453f98cba4dc51b56457533591f0739eee78ab Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 30 Mar 2026 17:54:31 +0100 Subject: [PATCH 23/24] Remove pal.h include from seqset test The seqset test is a unit test for ds_aal/seqset.h and only needs ds_aal and ds_core headers. With test/helpers.h no longer depending on DefaultPal, the pal.h include is unnecessary. --- src/test/func/seqset/seqset.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/func/seqset/seqset.cc b/src/test/func/seqset/seqset.cc index 36737f29f..429d8cd7a 100644 --- a/src/test/func/seqset/seqset.cc +++ b/src/test/func/seqset/seqset.cc @@ -10,7 +10,9 @@ * The corruption case is run in a forked child so that the expected abort * does not kill the test harness itself. */ -#include +#include +#include +#include #include #include From 98d7950000e1c2840ffeb38baa63777e2b3867bc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 30 Mar 2026 20:23:13 +0100 Subject: [PATCH 24/24] Apply clangformat Fix header includes that were missing. --- src/snmalloc/ds/ds.h | 2 +- src/snmalloc/ds/sizeclasstable.h | 12 +++---- src/snmalloc/global/globalalloc.h | 8 ++--- src/snmalloc/global/libc.h | 33 +++++++------------ src/snmalloc/mem/backend_concept.h | 1 + src/snmalloc/mem/corealloc.h | 8 ++--- src/snmalloc/mem/mem.h | 4 +-- src/snmalloc/mem/secondary/gwp_asan.h | 2 +- src/snmalloc/mitigations/allocconfig.h | 2 +- src/test/func/bits/bits.cc | 2 +- src/test/func/memory/memory.cc | 12 ++++--- src/test/func/multi_atexit/multi_atexit.cc | 2 +- .../multi_threadatexit/multi_threadatexit.cc | 2 +- src/test/func/protect_fork/protect_fork.cc | 2 +- .../perf/external_pointer/externalpointer.cc | 3 +- src/test/perf/large_alloc/large_alloc.cc | 2 +- src/test/perf/post_teardown/post-teardown.cc | 2 +- src/test/perf/singlethread/singlethread.cc | 6 ++-- 18 files changed, 48 insertions(+), 57 deletions(-) diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index 4a512c975..9c9cd0a49 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -10,6 +10,6 @@ #include "entropy.h" #include "mpmcstack.h" #include "pagemap.h" -#include "sizeclasstable.h" #include "pool.h" #include "pooled.h" +#include "sizeclasstable.h" diff --git a/src/snmalloc/ds/sizeclasstable.h b/src/snmalloc/ds/sizeclasstable.h index e09445b31..5db3cb5fa 100644 --- a/src/snmalloc/ds/sizeclasstable.h +++ b/src/snmalloc/ds/sizeclasstable.h @@ -181,8 +181,7 @@ namespace snmalloc { size_t max_capacity = 0; - for (smallsizeclass_t sizeclass(0); - sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { auto& meta = fast_small(sizeclass); @@ -213,8 +212,7 @@ namespace snmalloc // Get maximum precision to calculate largest division range. DIV_MULT_SHIFT = bits::BITS - bits::next_pow2_bits_const(max_capacity); - for (smallsizeclass_t sizeclass(0); - sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { // Calculate reciprocal division constant. @@ -417,7 +415,8 @@ namespace snmalloc sizeclass_compress_t sizeclass = 0; for (; sizeclass < minimum_class; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; + for (; curr <= + sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { table[sizeclass_lookup_index(curr)] = minimum_class; @@ -426,7 +425,8 @@ namespace snmalloc for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; + for (; curr <= + sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { auto i = sizeclass_lookup_index(curr); diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 748002b00..b102131d5 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -154,7 +154,7 @@ namespace snmalloc template size_t SNMALLOC_FAST_PATH_INLINE remaining_bytes(const void* p) { - return remaining_bytes(address_cast(p)); + return remaining_bytes(address_cast(p)); } /** @@ -382,8 +382,7 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_API void - dealloc(void* p, size_t size) + SNMALLOC_API void dealloc(void* p, size_t size) { check_size(p, size); ThreadAlloc::get().dealloc(p); @@ -396,8 +395,7 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_API void - dealloc(void* p, size_t size, size_t align) + SNMALLOC_API void dealloc(void* p, size_t size, size_t align) { auto rsize = aligned_size(align, size); check_size(p, rsize); diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index f80fa9d1d..a8e1b09e8 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -44,8 +44,7 @@ namespace snmalloc::libc * Allocate for a pre-computed small sizeclass. * Use is_small_sizeclass() + size_to_sizeclass_const() to get the class. */ - SNMALLOC_API void* - malloc_small(smallsizeclass_t sizeclass) + SNMALLOC_API void* malloc_small(smallsizeclass_t sizeclass) { return snmalloc::alloc(sizeclass); } @@ -53,8 +52,7 @@ namespace snmalloc::libc /** * Allocate zeroed memory for a pre-computed small sizeclass. */ - SNMALLOC_API void* - malloc_small_zero(smallsizeclass_t sizeclass) + SNMALLOC_API void* malloc_small_zero(smallsizeclass_t sizeclass) { return snmalloc::alloc(sizeclass); } @@ -64,20 +62,17 @@ namespace snmalloc::libc dealloc(ptr); } - SNMALLOC_API void - free_sized(void* ptr, size_t size) + SNMALLOC_API void free_sized(void* ptr, size_t size) { dealloc(ptr, size); } - SNMALLOC_API void - free_aligned_sized(void* ptr, size_t alignment, size_t size) + SNMALLOC_API void free_aligned_sized(void* ptr, size_t alignment, size_t size) { dealloc(ptr, size, alignment); } - SNMALLOC_API void* - calloc(size_t nmemb, size_t size) + SNMALLOC_API void* calloc(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -88,8 +83,7 @@ namespace snmalloc::libc return alloc(sz); } - SNMALLOC_API void* - realloc(void* ptr, size_t size) + SNMALLOC_API void* realloc(void* ptr, size_t size) { // Glibc treats // realloc(p, 0) as free(p) @@ -143,8 +137,7 @@ namespace snmalloc::libc return alloc_size(ptr); } - SNMALLOC_API_SLOW void* - reallocarray(void* ptr, size_t nmemb, size_t size) + SNMALLOC_API_SLOW void* reallocarray(void* ptr, size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -155,8 +148,7 @@ namespace snmalloc::libc return realloc(ptr, sz); } - SNMALLOC_API_SLOW int - reallocarr(void* ptr_, size_t nmemb, size_t size) + SNMALLOC_API_SLOW int reallocarr(void* ptr_, size_t nmemb, size_t size) { int err = errno; bool overflow = false; @@ -191,8 +183,7 @@ namespace snmalloc::libc return 0; } - SNMALLOC_API_SLOW void* - memalign(size_t alignment, size_t size) + SNMALLOC_API_SLOW void* memalign(size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY(alignment == 0 || !bits::is_pow2(alignment))) { @@ -202,8 +193,7 @@ namespace snmalloc::libc return alloc_aligned(alignment, size); } - SNMALLOC_API_SLOW void* - aligned_alloc(size_t alignment, size_t size) + SNMALLOC_API_SLOW void* aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } @@ -233,8 +223,7 @@ namespace snmalloc::libc * @tparam ReadsChecked also check the source */ template - SNMALLOC_API void* - memcpy(void* dst, const void* src, size_t len) + SNMALLOC_API void* memcpy(void* dst, const void* src, size_t len) { return snmalloc::memcpy(dst, src, len); } diff --git a/src/snmalloc/mem/backend_concept.h b/src/snmalloc/mem/backend_concept.h index 524b40b9a..03a566715 100644 --- a/src/snmalloc/mem/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -2,6 +2,7 @@ #ifdef __cpp_concepts # include "../ds/ds.h" +# include "../ds/sizeclasstable.h" # include diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 9bcee944d..eae5015ad 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -1,10 +1,10 @@ #pragma once #include "../ds/ds.h" +#include "../ds/pool.h" #include "check_init.h" #include "freelist.h" #include "metadata.h" -#include "../ds/pool.h" #include "remotecache.h" #include "snmalloc/stl/new.h" #include "ticker.h" @@ -610,8 +610,7 @@ namespace snmalloc { // Small allocations are more likely. Improve // branch prediction by placing this case first. - return small_alloc( - size_to_sizeclass(size), size); + return small_alloc(size_to_sizeclass(size), size); } return alloc_not_small(size, this); @@ -634,8 +633,7 @@ namespace snmalloc * Fast allocation for small objects, with pre-computed sizeclass. */ template - SNMALLOC_FAST_PATH void* - small_alloc( + SNMALLOC_FAST_PATH void* small_alloc( smallsizeclass_t sizeclass, size_t size) noexcept(noexcept(Conts::failure(0))) { diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index 6ee3caea6..fb0106a9f 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -1,5 +1,7 @@ #pragma once +#include "../ds/pool.h" +#include "../ds/pooled.h" #include "backend_concept.h" #include "backend_wrappers.h" #include "check_init.h" @@ -7,8 +9,6 @@ #include "entropy.h" #include "freelist.h" #include "metadata.h" -#include "../ds/pool.h" -#include "../ds/pooled.h" #include "remoteallocator.h" #include "remotecache.h" #include "ticker.h" diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index 897bbb17f..716c57935 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -1,8 +1,8 @@ #pragma once #include "gwp_asan/guarded_pool_allocator.h" -#include "snmalloc/ds_core/defines.h" #include "snmalloc/ds/sizeclasstable.h" +#include "snmalloc/ds_core/defines.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif diff --git a/src/snmalloc/mitigations/allocconfig.h b/src/snmalloc/mitigations/allocconfig.h index fcc8217d1..3f326a570 100644 --- a/src/snmalloc/mitigations/allocconfig.h +++ b/src/snmalloc/mitigations/allocconfig.h @@ -1,7 +1,7 @@ #pragma once -#include "mitigations.h" #include "../ds_core/sizeclassconfig.h" +#include "mitigations.h" namespace snmalloc { diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index 01deaaeea..d1ffcf90e 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -3,8 +3,8 @@ */ #include -#include #include +#include void test_ctz() { diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 08e101a15..9bf335087 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -394,7 +394,8 @@ void test_external_pointer_stack() if (snmalloc::libc::__malloc_start_pointer(&stack[i]) > &stack[i]) { std::cout << "Stack pointer: " << &stack[i] << " external pointer: " - << snmalloc::libc::__malloc_start_pointer(&stack[i]) << std::endl; + << snmalloc::libc::__malloc_start_pointer(&stack[i]) + << std::endl; abort(); } } @@ -408,7 +409,8 @@ void test_alloc_16M() const size_t size = 16'000'000; void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -418,7 +420,8 @@ void test_calloc_16M() const size_t size = 16'000'000; void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -431,7 +434,8 @@ void test_calloc_large_bug() const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } diff --git a/src/test/func/multi_atexit/multi_atexit.cc b/src/test/func/multi_atexit/multi_atexit.cc index d2e3978b7..09919ada3 100644 --- a/src/test/func/multi_atexit/multi_atexit.cc +++ b/src/test/func/multi_atexit/multi_atexit.cc @@ -13,8 +13,8 @@ #endif #ifdef RUN_TEST -# include # include +# include void do_nothing() {} diff --git a/src/test/func/multi_threadatexit/multi_threadatexit.cc b/src/test/func/multi_threadatexit/multi_threadatexit.cc index 10322b1a6..8e4ce1de3 100644 --- a/src/test/func/multi_threadatexit/multi_threadatexit.cc +++ b/src/test/func/multi_threadatexit/multi_threadatexit.cc @@ -15,8 +15,8 @@ #endif #ifdef RUN_TEST -# include # include +# include template void thread_destruct() diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index f008f7aca..aeade74f1 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -9,8 +9,8 @@ int main() # define SNMALLOC_PTHREAD_FORK_PROTECTION # include -# include # include +# include # include # include diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index df6f23ab3..07e69cef9 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -80,7 +80,8 @@ namespace test size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); - void* calced_external = snmalloc::libc::__malloc_start_pointer(interior_ptr); + void* calced_external = + snmalloc::libc::__malloc_start_pointer(interior_ptr); if (calced_external != external_ptr) { abort(); diff --git a/src/test/perf/large_alloc/large_alloc.cc b/src/test/perf/large_alloc/large_alloc.cc index 71cbdce80..b0f0f2bc8 100644 --- a/src/test/perf/large_alloc/large_alloc.cc +++ b/src/test/perf/large_alloc/large_alloc.cc @@ -1,6 +1,6 @@ -#include #include #include +#include using namespace snmalloc; diff --git a/src/test/perf/post_teardown/post-teardown.cc b/src/test/perf/post_teardown/post-teardown.cc index 4f42dd860..915513c08 100644 --- a/src/test/perf/post_teardown/post-teardown.cc +++ b/src/test/perf/post_teardown/post-teardown.cc @@ -1,7 +1,7 @@ #include -#include #include #include +#include #include using namespace snmalloc; diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index cad2620a2..bf173969d 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -1,6 +1,6 @@ -#include #include #include +#include #include using namespace snmalloc; @@ -11,8 +11,8 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) { MeasureTime m; m << "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) - << size - << ", ZeroMem: " << (zero_mem == ZeroMem::YesZero) << ", Write: " << write; + << size << ", ZeroMem: " << (zero_mem == ZeroMem::YesZero) + << ", Write: " << write; std::unordered_set set;