diff --git a/BUILD.bazel b/BUILD.bazel index 0d734ad4f..70af3d5f3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,6 +6,7 @@ filegroup( [ "src/snmalloc/**/*", "src/test/*.h", + "src/test/*.cc", "CMakeLists.txt", ], ), @@ -39,6 +40,7 @@ CMAKE_FLAGS = { "SNMALLOC_USE_SELF_VENDORED_STL": "OFF", "SNMALLOC_IPO": "ON", "USE_SNMALLOC_STATS": "ON", + "SNMALLOC_BUILD_TESTING": "OFF", } | select({ ":release_with_debug": {"CMAKE_BUILD_TYPE": "RelWithDebInfo"}, ":release": {"CMAKE_BUILD_TYPE": "Release"}, diff --git a/CMakeLists.txt b/CMakeLists.txt index e21d6bd71..f1c7e7153 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,16 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(TEST_CLEANUP ${SNMALLOC_CLEANUP}) endif() + # Tests that only include snmalloc_testlib.h (no snmalloc.h or snmalloc_core.h). + # These are mitigation-independent and can be compiled once, then linked + # against both fast and check testlib variants. + set(TESTLIB_ONLY_TESTS + bits first_operation memory memory_usage multi_atexit multi_threadatexit + redblack statistics teardown + contention external_pointer large_alloc lotsofthreads post_teardown + singlethread startup + ) + function(make_tests TAG DEFINES) foreach(TEST_CATEGORY ${TEST_CATEGORIES}) message(VERBOSE "Adding ${TAG}/${TEST_CATEGORY} tests") @@ -488,7 +498,32 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC) set(TESTNAME "${TEST_CATEGORY}-${TEST}-${TAG}") - add_executable(${TESTNAME} ${SRC}) + # Check if this test is testlib-only (mitigation-independent). + # If so, compile the source once into an object library and reuse + # across flavours. + list(FIND TESTLIB_ONLY_TESTS ${TEST} _testlib_only_idx) + set(OBJLIBNAME "${TEST_CATEGORY}-${TEST}-obj") + if (NOT ${_testlib_only_idx} EQUAL -1 AND NOT TARGET ${OBJLIBNAME}) + # First flavour to see this test creates the object library. + add_library(${OBJLIBNAME} OBJECT ${SRC}) + target_link_libraries(${OBJLIBNAME} snmalloc) + target_compile_definitions(${OBJLIBNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") + add_warning_flags(${OBJLIBNAME}) + if(SNMALLOC_SANITIZER) + target_compile_options(${OBJLIBNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) + if (${SNMALLOC_SANITIZER} MATCHES "thread") + target_compile_definitions(${OBJLIBNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED) + endif() + endif() + endif() + + if (NOT ${_testlib_only_idx} EQUAL -1) + # Testlib-only: link the shared object library against this + # flavour's testlib. + add_executable(${TESTNAME} $) + else() + add_executable(${TESTNAME} ${SRC}) + endif() if(SNMALLOC_SANITIZER) target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) @@ -501,6 +536,10 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) add_warning_flags(${TESTNAME}) target_link_libraries(${TESTNAME} snmalloc) + # Link the pre-compiled test library into all tests. For tests + # that include snmalloc.h directly the linker will simply discard + # the duplicate inline definitions (ODR-safe). + target_link_libraries(${TESTNAME} snmalloc-testlib-${TAG}) target_compile_definitions(${TESTNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") if (NOT DEFINES STREQUAL " ") @@ -538,6 +577,24 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endforeach() endfunction() + function(build_test_library TAG DEFINES) + set(LIBNAME snmalloc-testlib-${TAG}) + add_library(${LIBNAME} STATIC src/test/snmalloc_testlib.cc) + target_link_libraries(${LIBNAME} PRIVATE snmalloc) + set_target_properties(${LIBNAME} PROPERTIES INTERFACE_LINK_LIBRARIES "") + target_compile_definitions(${LIBNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") + if (NOT DEFINES STREQUAL " ") + target_compile_definitions(${LIBNAME} PRIVATE ${DEFINES}) + endif() + if(SNMALLOC_SANITIZER) + target_compile_options(${LIBNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) + if (${SNMALLOC_SANITIZER} MATCHES "thread") + target_compile_definitions(${LIBNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED) + endif() + endif() + add_warning_flags(${LIBNAME}) + endfunction() + if(NOT (DEFINED SNMALLOC_LINKER_FLAVOUR) OR ("${SNMALLOC_LINKER_FLAVOUR}" MATCHES "^$")) # Linker not specified externally; probe to see if we can make lld work set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld -Wl,--icf=all) @@ -680,6 +737,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(DEFINES " ") endif() + build_test_library(${FLAVOUR} ${DEFINES}) make_tests(${FLAVOUR} ${DEFINES}) endforeach() endif() @@ -743,6 +801,7 @@ install(DIRECTORY src/snmalloc/ds_aal DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/ds_core DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/global DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/mem DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/mitigations DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/override DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/pal DESTINATION include/snmalloc) install(DIRECTORY src/snmalloc/stl DESTINATION include/snmalloc) diff --git a/src/snmalloc/README.md b/src/snmalloc/README.md index 9f593ac1d..2549320fb 100644 --- a/src/snmalloc/README.md +++ b/src/snmalloc/README.md @@ -10,6 +10,9 @@ These are arranged in a hierarchy such that each of the directories may include This layer provides abstractions over CPU-specific intrinsics and defines things such as the virtual address-space size. There is a single AAL for an snmalloc instantiation. - `ds_aal/` provides data structures that depend on the AAL. + - `mitigations/` provides compile-time configuration for security mitigations. + This includes the `mitigations()` function (controlled by `SNMALLOC_CHECK_CLIENT`), mitigation-dependent allocator constants, and CHERI capability checks. + Layers below this (`ds_core/`, `aal/`, `ds_aal/`) are mitigation-independent, which allows code that only includes those layers to be compiled once regardless of mitigation settings. - `pal/` provides the platform abstraction layer (PAL). This exposes OS- or environment-specific abstractions into the rest of the code. An snmalloc instantiation may use more than one PAL, including ones provided by the user. diff --git a/src/snmalloc/ds/allocconfig.h b/src/snmalloc/ds/allocconfig.h deleted file mode 100644 index d58d863b8..000000000 --- a/src/snmalloc/ds/allocconfig.h +++ /dev/null @@ -1,200 +0,0 @@ -#pragma once - -namespace snmalloc -{ - // 0 intermediate bits results in power of 2 small allocs. 1 intermediate - // bit gives additional sizeclasses at the midpoint between each power of 2. - // 2 intermediate bits gives 3 intermediate sizeclasses, etc. - static constexpr size_t INTERMEDIATE_BITS = -#ifdef USE_INTERMEDIATE_BITS - USE_INTERMEDIATE_BITS -#else - 2 -#endif - ; - - // The remaining values are derived, not configurable. - static constexpr size_t POINTER_BITS = - bits::next_pow2_bits_const(sizeof(uintptr_t)); - - // Used to isolate values on cache lines to prevent false sharing. - static constexpr size_t CACHELINE_SIZE = 64; - - /// The "machine epsilon" for the small sizeclass machinery. - static constexpr size_t MIN_ALLOC_STEP_SIZE = -#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE) - SNMALLOC_MIN_ALLOC_STEP_SIZE; -#else - 2 * sizeof(void*); -#endif - - /// Derived from MIN_ALLOC_STEP_SIZE - static constexpr size_t MIN_ALLOC_STEP_BITS = - bits::ctz_const(MIN_ALLOC_STEP_SIZE); - static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE)); - - /** - * Minimum allocation size is space for two pointers. If the small sizeclass - * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is - * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must - * be large or not a power of two, those smaller size classes will be unused. - */ - static constexpr size_t MIN_ALLOC_SIZE = -#if defined(SNMALLOC_MIN_ALLOC_SIZE) - SNMALLOC_MIN_ALLOC_SIZE; -#else - 2 * sizeof(void*); -#endif - - // Minimum slab size. -#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) - /* - * QEMU user-mode, up through and including v7.2.0-rc4, the latest tag at the - * time of this writing, does not use a tree of any sort to store its opinion - * of the address space, allocating an amount of memory linear in the size of - * any created map, not the number of pages actually used. This is - * exacerbated in and after qemu v6 (or, more specifically, d9c58585), which - * grew the proportionality constant. - * - * In any case, for our CI jobs, then, use a larger minimum chunk size (that - * is, pagemap granularity) than by default to reduce the size of the - * pagemap. We can't raise this *too* much, lest we hit constexpr step - * limits in the sizeclasstable magic! 17 bits seems to be the sweet spot - * and means that any of our tests can run in a little under 2 GiB of RSS - * even on QEMU versions after v6. - */ - static constexpr size_t MIN_CHUNK_BITS = static_cast(17); -#else - static constexpr size_t MIN_CHUNK_BITS = static_cast(14); -#endif - static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); - - // Minimum number of objects on a slab - static constexpr size_t MIN_OBJECT_COUNT = - mitigations(random_larger_thresholds) ? 13 : 4; - - // Maximum size of an object that uses sizeclasses. -#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) - /* - * As a consequence of our significantly larger minimum chunk size, we need - * to raise the threshold for what constitutes a large object (which must - * be a multiple of the minimum chunk size). Extend the space of small - * objects up enough to match yet preserve the notion that there exist small - * objects larger than MIN_CHUNK_SIZE. - */ - static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 19; -#else - static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 16; -#endif - static constexpr size_t MAX_SMALL_SIZECLASS_SIZE = - bits::one_at_bit(MAX_SMALL_SIZECLASS_BITS); - - static_assert( - MAX_SMALL_SIZECLASS_SIZE >= MIN_CHUNK_SIZE, - "Large sizes need to be representable by as a multiple of MIN_CHUNK_SIZE"); - - /** - * The number of bits needed to count the number of objects within a slab. - * - * Most likely, this is achieved by the smallest sizeclass, which will have - * many more than MIN_OBJECT_COUNT objects in its slab. But, just in case, - * it's defined here and checked when we compute the sizeclass table, since - * computing this number is potentially nontrivial. - */ -#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) - static constexpr size_t MAX_CAPACITY_BITS = 13; -#else - static constexpr size_t MAX_CAPACITY_BITS = 11; -#endif - - /** - * The maximum distance between the start of two objects in the same slab. - */ - static constexpr size_t MAX_SLAB_SPAN_SIZE = - (MIN_OBJECT_COUNT - 1) * MAX_SMALL_SIZECLASS_SIZE; - static constexpr size_t MAX_SLAB_SPAN_BITS = - bits::next_pow2_bits_const(MAX_SLAB_SPAN_SIZE); - - // Number of slots for remote deallocation. - static constexpr size_t REMOTE_SLOT_BITS = 8; - static constexpr size_t REMOTE_SLOTS = 1 << REMOTE_SLOT_BITS; - static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1; - -#if defined(SNMALLOC_DEALLOC_BATCH_RING_ASSOC) - static constexpr size_t DEALLOC_BATCH_RING_ASSOC = - SNMALLOC_DEALLOC_BATCH_RING_ASSOC; -#else -# if defined(__has_cpp_attribute) -# if ( \ - __has_cpp_attribute(msvc::no_unique_address) && \ - (__cplusplus >= 201803L || _MSVC_LANG >= 201803L)) || \ - __has_cpp_attribute(no_unique_address) - // For C++20 or later, we do have [[no_unique_address]] and so can also do - // batching if we aren't turning on the backward-pointer mitigations - static constexpr size_t DEALLOC_BATCH_MIN_ALLOC_WORDS = - mitigations(freelist_backward_edge) ? 4 : 2; -# else - // For C++17, we don't have [[no_unique_address]] and so we always end up - // needing all four pointers' worth of space (because BatchedRemoteMessage has - // two freelist::Object::T<> links within, each of which will have two fields - // and will be padded to two pointers). - static constexpr size_t DEALLOC_BATCH_MIN_ALLOC_WORDS = 4; -# endif -# else - // If we don't even have the feature test macro, we're C++17 or earlier. - static constexpr size_t DEALLOC_BATCH_MIN_ALLOC_WORDS = 4; -# endif - - static constexpr size_t DEALLOC_BATCH_RING_ASSOC = - (MIN_ALLOC_SIZE >= (DEALLOC_BATCH_MIN_ALLOC_WORDS * sizeof(void*))) ? 2 : 0; -#endif - -#if defined(SNMALLOC_DEALLOC_BATCH_RING_SET_BITS) - static constexpr size_t DEALLOC_BATCH_RING_SET_BITS = - SNMALLOC_DEALLOC_BATCH_RING_SET_BITS; -#else - static constexpr size_t DEALLOC_BATCH_RING_SET_BITS = 3; -#endif - - static constexpr size_t DEALLOC_BATCH_RINGS = - DEALLOC_BATCH_RING_ASSOC * bits::one_at_bit(DEALLOC_BATCH_RING_SET_BITS); - - static_assert( - INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS, - "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); - static_assert( - MIN_ALLOC_SIZE >= (sizeof(void*) * 2), - "MIN_ALLOC_SIZE must be sufficient for two pointers"); - static_assert( - 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >= - bits::next_pow2_const(MIN_ALLOC_SIZE), - "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE"); - static_assert( - MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE, - "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE"); - - // Return remote small allocs when the local cache reaches this size. - static constexpr int64_t REMOTE_CACHE = -#ifdef USE_REMOTE_CACHE - USE_REMOTE_CACHE -#else - MIN_CHUNK_SIZE -#endif - ; - - // Stop processing remote batch when we reach this amount of deallocations in - // bytes - static constexpr int64_t REMOTE_BATCH_LIMIT = -#ifdef SNMALLOC_REMOTE_BATCH_PROCESS_SIZE - SNMALLOC_REMOTE_BATCH_PROCESS_SIZE -#else - 1 * 1024 * 1024 -#endif - ; - - // Used to configure when the backend should use thread local buddies. - // This only basically is used to disable some buddy allocators on small - // fixed heap scenarios like OpenEnclave. - static constexpr size_t MIN_HEAP_SIZE_FOR_THREAD_LOCAL_BUDDY = - bits::one_at_bit(27); -} // namespace snmalloc diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index e24b1f25b..9c9cd0a49 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -6,8 +6,10 @@ #include "../ds_aal/ds_aal.h" #include "../pal/pal.h" #include "aba.h" -#include "allocconfig.h" #include "combininglock.h" #include "entropy.h" #include "mpmcstack.h" #include "pagemap.h" +#include "pool.h" +#include "pooled.h" +#include "sizeclasstable.h" diff --git a/src/snmalloc/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h index f1e965e53..57e587637 100644 --- a/src/snmalloc/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -2,7 +2,6 @@ #include "../ds_core/ds_core.h" #include "aba.h" -#include "allocconfig.h" namespace snmalloc { diff --git a/src/snmalloc/mem/pool.h b/src/snmalloc/ds/pool.h similarity index 100% rename from src/snmalloc/mem/pool.h rename to src/snmalloc/ds/pool.h diff --git a/src/snmalloc/mem/pooled.h b/src/snmalloc/ds/pooled.h similarity index 94% rename from src/snmalloc/mem/pooled.h rename to src/snmalloc/ds/pooled.h index ca4e94101..d64716783 100644 --- a/src/snmalloc/mem/pooled.h +++ b/src/snmalloc/ds/pooled.h @@ -1,7 +1,7 @@ #pragma once #include "../ds/ds.h" -#include "backend_concept.h" +#include "../mem/backend_concept.h" namespace snmalloc { diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/ds/sizeclasstable.h similarity index 92% rename from src/snmalloc/mem/sizeclasstable.h rename to src/snmalloc/ds/sizeclasstable.h index d31a1f692..5db3cb5fa 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/ds/sizeclasstable.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/ds.h" +#include "../pal/pal.h" /** * This file contains all the code for transforming transforming sizes to @@ -15,25 +15,8 @@ namespace snmalloc { - using smallsizeclass_t = size_t; using chunksizeclass_t = size_t; - static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) - { - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - auto sc = static_cast( - bits::to_exp_mant_const(size)); - - SNMALLOC_ASSERT(sc == static_cast(sc)); - - return sc; - } - - constexpr size_t NUM_SMALL_SIZECLASSES = - size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE) + 1; - // Large classes range from [MAX_SMALL_SIZECLASS_SIZE, ADDRESS_SPACE). constexpr size_t NUM_LARGE_CLASSES = DefaultPal::address_bits - MAX_SMALL_SIZECLASS_BITS; @@ -97,7 +80,7 @@ namespace snmalloc constexpr smallsizeclass_t as_small() { SNMALLOC_ASSERT(is_small()); - return value & (TAG - 1); + return smallsizeclass_t(value & (TAG - 1)); } constexpr chunksizeclass_t as_large() @@ -198,8 +181,7 @@ namespace snmalloc { size_t max_capacity = 0; - for (sizeclass_compress_t sizeclass = 0; - sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { auto& meta = fast_small(sizeclass); @@ -230,8 +212,7 @@ namespace snmalloc // Get maximum precision to calculate largest division range. DIV_MULT_SHIFT = bits::BITS - bits::next_pow2_bits_const(max_capacity); - for (sizeclass_compress_t sizeclass = 0; - sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { // Calculate reciprocal division constant. @@ -434,7 +415,8 @@ namespace snmalloc sizeclass_compress_t sizeclass = 0; for (; sizeclass < minimum_class; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + for (; curr <= + sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { table[sizeclass_lookup_index(curr)] = minimum_class; @@ -443,7 +425,8 @@ namespace snmalloc for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + for (; curr <= + sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { auto i = sizeclass_lookup_index(curr); @@ -457,30 +440,19 @@ namespace snmalloc constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup(); - /** - * @brief Returns true if the size is a small sizeclass. Note that - * 0 is not considered a small sizeclass. - */ - constexpr bool is_small_sizeclass(size_t size) - { - // Perform the - 1 on size, so that zero wraps around and ends up on - // slow path. - return (size - 1) < sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1); - } - constexpr smallsizeclass_t size_to_sizeclass(size_t size) { if (SNMALLOC_LIKELY(is_small_sizeclass(size))) { auto index = sizeclass_lookup_index(size); SNMALLOC_ASSERT(index < sizeclass_lookup_size); - return sizeclass_lookup.table[index]; + return smallsizeclass_t(sizeclass_lookup.table[index]); } // Check this is not called on large sizes. SNMALLOC_ASSERT(size == 0); // Map size == 0 to the first sizeclass. - return 0; + return smallsizeclass_t(0); } /** diff --git a/src/snmalloc/ds_core/defines.h b/src/snmalloc/ds_core/defines.h index 8c83210ce..45cea3d70 100644 --- a/src/snmalloc/ds_core/defines.h +++ b/src/snmalloc/ds_core/defines.h @@ -123,8 +123,11 @@ namespace snmalloc static constexpr bool Debug = true; #endif - // Forwards reference so that the platform can define how to handle errors. + // Forward references so that the platform can define how to handle + // errors and messages. Definitions are provided in pal/pal.h once + // DefaultPal is known. [[noreturn]] SNMALLOC_COLD void error(const char* const str); + void message_impl(const char* const str); } // namespace snmalloc #define TOSTRING(expr) TOSTRING2(expr) @@ -213,21 +216,21 @@ namespace snmalloc namespace snmalloc { + template + SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) + {} + /** - * Forward declaration so that this can be called before the pal header is - * included. + * Forward declaration so that this can be called before helpers.h is + * included (e.g. in SNMALLOC_ASSERT macros expanded inside bits.h). */ template [[noreturn]] inline void report_fatal_error(Args... args); /** - * Forward declaration so that this can be called before the pal header is + * Forward declaration so that this can be called before helpers.h is * included. */ template inline void message(Args... args); - - template - SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) - {} } // namespace snmalloc diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index cc395127b..0c22a35bd 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -8,11 +8,11 @@ */ #include "bits.h" -#include "cheri.h" #include "concept.h" #include "defines.h" #include "helpers.h" -#include "mitigations.h" #include "ptrwrap.h" #include "redblacktree.h" +#include "sizeclassconfig.h" +#include "sizeclassstatic.h" #include "tid.h" \ No newline at end of file diff --git a/src/snmalloc/ds_core/helpers.h b/src/snmalloc/ds_core/helpers.h index e5a0ca00e..7bddbfa94 100644 --- a/src/snmalloc/ds_core/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -2,6 +2,7 @@ #include "bits.h" #include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/tid.h" #include "snmalloc/stl/array.h" #include "snmalloc/stl/type_traits.h" #include "snmalloc/stl/utility.h" @@ -339,6 +340,38 @@ namespace snmalloc } }; + /** + * Report a fatal error via a PAL-specific error reporting mechanism. This + * takes a format string and a set of arguments. The format string indicates + * the remaining arguments with "{}". This could be extended later to + * support indexing fairly easily, if we ever want to localise these error + * messages. + * + * The following are supported as arguments: + * + * - Characters (`char`), printed verbatim. + * - Strings Literals (`const char*` or `const char[]`), printed verbatim. + * - Raw pointers (void*), printed as hex strings. + * - Integers (convertible to `size_t`), printed as hex strings. + * + * These types should be sufficient for allocator-related error messages. + */ + template + [[noreturn]] inline void report_fatal_error(Args... args) + { + MessageBuilder msg{stl::forward(args)...}; + error(msg.get_message()); + } + + template + inline void message(Args... args) + { + MessageBuilder msg{stl::forward(args)...}; + MessageBuilder msg_tid{ + "{}: {}", debug_get_tid(), msg.get_message()}; + message_impl(msg_tid.get_message()); + } + /** * Convenience type that has no fields / methods. */ diff --git a/src/snmalloc/ds_core/sizeclassconfig.h b/src/snmalloc/ds_core/sizeclassconfig.h new file mode 100644 index 000000000..05224f81f --- /dev/null +++ b/src/snmalloc/ds_core/sizeclassconfig.h @@ -0,0 +1,85 @@ +#pragma once + +#include "bits.h" + +namespace snmalloc +{ + // 0 intermediate bits results in power of 2 small allocs. 1 intermediate + // bit gives additional sizeclasses at the midpoint between each power of 2. + // 2 intermediate bits gives 3 intermediate sizeclasses, etc. + static constexpr size_t INTERMEDIATE_BITS = +#ifdef USE_INTERMEDIATE_BITS + USE_INTERMEDIATE_BITS +#else + 2 +#endif + ; + + // The remaining values are derived, not configurable. + static constexpr size_t POINTER_BITS = + bits::next_pow2_bits_const(sizeof(uintptr_t)); + + // Used to isolate values on cache lines to prevent false sharing. + static constexpr size_t CACHELINE_SIZE = 64; + + /// The "machine epsilon" for the small sizeclass machinery. + static constexpr size_t MIN_ALLOC_STEP_SIZE = +#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE) + SNMALLOC_MIN_ALLOC_STEP_SIZE; +#else + 2 * sizeof(void*); +#endif + + /// Derived from MIN_ALLOC_STEP_SIZE + static constexpr size_t MIN_ALLOC_STEP_BITS = + bits::ctz_const(MIN_ALLOC_STEP_SIZE); + static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE)); + + /** + * Minimum allocation size is space for two pointers. If the small sizeclass + * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is + * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must + * be large or not a power of two, those smaller size classes will be unused. + */ + static constexpr size_t MIN_ALLOC_SIZE = +#if defined(SNMALLOC_MIN_ALLOC_SIZE) + SNMALLOC_MIN_ALLOC_SIZE; +#else + 2 * sizeof(void*); +#endif + + // Minimum slab size. +#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) + static constexpr size_t MIN_CHUNK_BITS = static_cast(17); +#else + static constexpr size_t MIN_CHUNK_BITS = static_cast(14); +#endif + static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); + + // Maximum size of an object that uses sizeclasses. +#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) + static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 19; +#else + static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 16; +#endif + static constexpr size_t MAX_SMALL_SIZECLASS_SIZE = + bits::one_at_bit(MAX_SMALL_SIZECLASS_BITS); + + static_assert( + MAX_SMALL_SIZECLASS_SIZE >= MIN_CHUNK_SIZE, + "Large sizes need to be representable by as a multiple of MIN_CHUNK_SIZE"); + + static_assert( + INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS, + "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); + static_assert( + MIN_ALLOC_SIZE >= (sizeof(void*) * 2), + "MIN_ALLOC_SIZE must be sufficient for two pointers"); + static_assert( + 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >= + bits::next_pow2_const(MIN_ALLOC_SIZE), + "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE"); + static_assert( + MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE, + "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE"); +} // namespace snmalloc diff --git a/src/snmalloc/ds_core/sizeclassstatic.h b/src/snmalloc/ds_core/sizeclassstatic.h new file mode 100644 index 000000000..95cacf01b --- /dev/null +++ b/src/snmalloc/ds_core/sizeclassstatic.h @@ -0,0 +1,72 @@ +#pragma once + +#include "sizeclassconfig.h" + +namespace snmalloc +{ + /** + * A wrapper type for small sizeclass indices. + * + * Implicitly converts TO size_t (for array indexing, comparisons, etc.) + * but does NOT implicitly convert FROM size_t — construction must be + * explicit via smallsizeclass_t(value). + */ + struct smallsizeclass_t + { + size_t raw{0}; + + constexpr smallsizeclass_t() = default; + + explicit constexpr smallsizeclass_t(size_t v) : raw(v) {} + + /// Implicit conversion to size_t. + constexpr operator size_t() const + { + return raw; + } + + /// Pre-increment. + constexpr smallsizeclass_t& operator++() + { + ++raw; + return *this; + } + + /// Post-increment. + constexpr smallsizeclass_t operator++(int) + { + auto tmp = *this; + ++raw; + return tmp; + } + }; + + static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) + { + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + return smallsizeclass_t( + bits::to_exp_mant_const(size)); + } + + constexpr size_t NUM_SMALL_SIZECLASSES = + size_t(size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE)) + 1; + + static constexpr size_t sizeclass_to_size_const(smallsizeclass_t sc) + { + return bits::from_exp_mant(sc); + } + + /** + * @brief Returns true if the size is a small sizeclass. Note that + * 0 is not considered a small sizeclass. + */ + constexpr bool is_small_sizeclass(size_t size) + { + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + return (size - 1) < + sizeclass_to_size_const(smallsizeclass_t(NUM_SMALL_SIZECLASSES - 1)); + } +} // namespace snmalloc diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 110051e2f..b102131d5 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -3,6 +3,14 @@ #include "../mem/mem.h" #include "threadalloc.h" +// Non-template API functions use SNMALLOC_API for their linkage specifier. +// Normally inline; can be overridden to NOINLINE (via SNMALLOC_API_NOINLINE) +// to produce strong definitions in a standalone compilation unit. +#ifndef SNMALLOC_API +# define SNMALLOC_API SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE +# define SNMALLOC_API_SLOW SNMALLOC_USED_FUNCTION inline +#endif + namespace snmalloc { template @@ -17,19 +25,19 @@ namespace snmalloc // Handling the message queue for each stack is non-atomic. auto* first = AllocPool::extract(); auto* alloc = first; - decltype(alloc) last; - if (alloc != nullptr) - { - while (alloc != nullptr) - { - alloc->flush(); - last = alloc; - alloc = AllocPool::extract(alloc); - } + if (alloc == nullptr) + return; - AllocPool::restore(first, last); + decltype(alloc) last = alloc; + while (alloc != nullptr) + { + alloc->flush(); + last = alloc; + alloc = AllocPool::extract(alloc); } + + AllocPool::restore(first, last); } /** @@ -143,6 +151,12 @@ namespace snmalloc return snmalloc::remaining_bytes(sizeclass, p); } + template + size_t SNMALLOC_FAST_PATH_INLINE remaining_bytes(const void* p) + { + return remaining_bytes(address_cast(p)); + } + /** * Returns the byte offset into an object. * @@ -324,8 +338,18 @@ namespace snmalloc template SNMALLOC_FAST_PATH_INLINE void* alloc() { - return ThreadAlloc::get().alloc( - aligned_size(align, size)); + constexpr size_t sz = aligned_size(align, size); + if constexpr (is_small_sizeclass(sz)) + { + constexpr auto sc = size_to_sizeclass_const(sz); + return ThreadAlloc::get().template alloc( + sc); + } + else + { + return ThreadAlloc::get().template alloc( + sz); + } } template @@ -335,6 +359,17 @@ namespace snmalloc aligned_size(align, size)); } + /** + * Allocate a block for a known small sizeclass. + * The sizeclass can be computed at compile time with size_to_sizeclass_const. + */ + template + SNMALLOC_FAST_PATH_INLINE void* alloc(smallsizeclass_t sizeclass) + { + return ThreadAlloc::get().template alloc( + sizeclass); + } + template SNMALLOC_FAST_PATH_INLINE void* alloc_aligned(size_t align, size_t size) { @@ -342,12 +377,12 @@ namespace snmalloc aligned_size(align, size)); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) + SNMALLOC_API void dealloc(void* p) { ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size) + SNMALLOC_API void dealloc(void* p, size_t size) { check_size(p, size); ThreadAlloc::get().dealloc(p); @@ -360,14 +395,14 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size, size_t align) + SNMALLOC_API void dealloc(void* p, size_t size, size_t align) { auto rsize = aligned_size(align, size); check_size(p, rsize); ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void debug_teardown() + SNMALLOC_API void debug_teardown() { return ThreadAlloc::teardown(); } diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index 726f61190..a8e1b09e8 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -1,6 +1,7 @@ #pragma once #include "globalalloc.h" +#include "memcpy.h" #include #include @@ -19,33 +20,59 @@ namespace snmalloc::libc return err; } - inline void* __malloc_end_pointer(void* ptr) + SNMALLOC_API_SLOW void* __malloc_end_pointer(void* ptr) { return snmalloc::external_pointer(ptr); } - SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) + SNMALLOC_API_SLOW void* __malloc_start_pointer(void* ptr) + { + return snmalloc::external_pointer(ptr); + } + + SNMALLOC_API_SLOW void* __malloc_last_byte_pointer(void* ptr) + { + return snmalloc::external_pointer(ptr); + } + + SNMALLOC_API void* malloc(size_t size) { return snmalloc::alloc(size); } - SNMALLOC_FAST_PATH_INLINE void free(void* ptr) + /** + * Allocate for a pre-computed small sizeclass. + * Use is_small_sizeclass() + size_to_sizeclass_const() to get the class. + */ + SNMALLOC_API void* malloc_small(smallsizeclass_t sizeclass) + { + return snmalloc::alloc(sizeclass); + } + + /** + * Allocate zeroed memory for a pre-computed small sizeclass. + */ + SNMALLOC_API void* malloc_small_zero(smallsizeclass_t sizeclass) + { + return snmalloc::alloc(sizeclass); + } + + SNMALLOC_API void free(void* ptr) { dealloc(ptr); } - SNMALLOC_FAST_PATH_INLINE void free_sized(void* ptr, size_t size) + SNMALLOC_API void free_sized(void* ptr, size_t size) { dealloc(ptr, size); } - SNMALLOC_FAST_PATH_INLINE void - free_aligned_sized(void* ptr, size_t alignment, size_t size) + SNMALLOC_API void free_aligned_sized(void* ptr, size_t alignment, size_t size) { dealloc(ptr, size, alignment); } - SNMALLOC_FAST_PATH_INLINE void* calloc(size_t nmemb, size_t size) + SNMALLOC_API void* calloc(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -56,7 +83,7 @@ namespace snmalloc::libc return alloc(sz); } - SNMALLOC_FAST_PATH_INLINE void* realloc(void* ptr, size_t size) + SNMALLOC_API void* realloc(void* ptr, size_t size) { // Glibc treats // realloc(p, 0) as free(p) @@ -105,12 +132,12 @@ namespace snmalloc::libc return p; } - inline size_t malloc_usable_size(const void* ptr) + SNMALLOC_API_SLOW size_t malloc_usable_size(const void* ptr) { return alloc_size(ptr); } - inline void* reallocarray(void* ptr, size_t nmemb, size_t size) + SNMALLOC_API_SLOW void* reallocarray(void* ptr, size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -121,7 +148,7 @@ namespace snmalloc::libc return realloc(ptr, sz); } - inline int reallocarr(void* ptr_, size_t nmemb, size_t size) + SNMALLOC_API_SLOW int reallocarr(void* ptr_, size_t nmemb, size_t size) { int err = errno; bool overflow = false; @@ -156,7 +183,7 @@ namespace snmalloc::libc return 0; } - inline void* memalign(size_t alignment, size_t size) + SNMALLOC_API_SLOW void* memalign(size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY(alignment == 0 || !bits::is_pow2(alignment))) { @@ -166,12 +193,13 @@ namespace snmalloc::libc return alloc_aligned(alignment, size); } - inline void* aligned_alloc(size_t alignment, size_t size) + SNMALLOC_API_SLOW void* aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } - inline int posix_memalign(void** memptr, size_t alignment, size_t size) + SNMALLOC_API_SLOW int + posix_memalign(void** memptr, size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY( (alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment)))) @@ -188,4 +216,15 @@ namespace snmalloc::libc *memptr = p; return 0; } + + /** + * Checked memcpy through the libc API surface. + * @tparam Checked check the destination is within a snmalloc allocation + * @tparam ReadsChecked also check the source + */ + template + SNMALLOC_API void* memcpy(void* dst, const void* src, size_t len) + { + return snmalloc::memcpy(dst, src, len); + } } // namespace snmalloc::libc diff --git a/src/snmalloc/mem/backend_concept.h b/src/snmalloc/mem/backend_concept.h index 9e0a21e77..03a566715 100644 --- a/src/snmalloc/mem/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -2,7 +2,7 @@ #ifdef __cpp_concepts # include "../ds/ds.h" -# include "sizeclasstable.h" +# include "../ds/sizeclasstable.h" # include diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 5ec7bf1f3..eae5015ad 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -1,12 +1,11 @@ #pragma once #include "../ds/ds.h" +#include "../ds/pool.h" #include "check_init.h" #include "freelist.h" #include "metadata.h" -#include "pool.h" #include "remotecache.h" -#include "sizeclasstable.h" #include "snmalloc/stl/new.h" #include "ticker.h" @@ -607,23 +606,36 @@ namespace snmalloc { // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (SNMALLOC_LIKELY( - (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) + if (SNMALLOC_LIKELY(is_small_sizeclass(size))) { // Small allocations are more likely. Improve // branch prediction by placing this case first. - return small_alloc(size); + return small_alloc(size_to_sizeclass(size), size); } return alloc_not_small(size, this); } /** - * Fast allocation for small objects. + * Allocates a block of memory for a known small sizeclass. + * Callers can pre-compute the sizeclass (e.g. at compile time with + * size_to_sizeclass_const) and avoid the dynamic sizeclass lookup. + */ + template + SNMALLOC_FAST_PATH ALLOCATOR void* + alloc(smallsizeclass_t sizeclass) noexcept(noexcept(Conts::failure(0))) + { + return small_alloc( + sizeclass, sizeclass_to_size(sizeclass)); + } + + /** + * Fast allocation for small objects, with pre-computed sizeclass. */ template - SNMALLOC_FAST_PATH void* - small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) + SNMALLOC_FAST_PATH void* small_alloc( + smallsizeclass_t sizeclass, + size_t size) noexcept(noexcept(Conts::failure(0))) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { @@ -631,7 +643,6 @@ namespace snmalloc }; auto& key = freelist::Object::key_root; - smallsizeclass_t sizeclass = size_to_sizeclass(size); auto* fl = &small_fast_free_lists[sizeclass]; if (SNMALLOC_LIKELY(!fl->empty())) { @@ -653,6 +664,17 @@ namespace snmalloc size); } + /** + * Fast allocation for small objects from a byte size. + * Computes the sizeclass and delegates to the two-arg version. + */ + template + SNMALLOC_FAST_PATH void* + small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) + { + return small_alloc(size_to_sizeclass(size), size); + } + /** * Allocation that are larger that will result in an allocation directly * from the backend. This additionally checks for 0 size allocations and @@ -1400,7 +1422,7 @@ namespace snmalloc auto& key = freelist::Object::key_root; - for (size_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) + for (smallsizeclass_t i; i < NUM_SMALL_SIZECLASSES; i++) { if (small_fast_free_lists[i].empty()) continue; @@ -1421,7 +1443,7 @@ namespace snmalloc local_state, get_trunc_id()); // We may now have unused slabs, return to the global allocator. - for (smallsizeclass_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { dealloc_local_slabs(sizeclass); diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index 3e9212635..fb0106a9f 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -1,5 +1,7 @@ #pragma once +#include "../ds/pool.h" +#include "../ds/pooled.h" #include "backend_concept.h" #include "backend_wrappers.h" #include "check_init.h" @@ -7,9 +9,6 @@ #include "entropy.h" #include "freelist.h" #include "metadata.h" -#include "pool.h" -#include "pooled.h" #include "remoteallocator.h" #include "remotecache.h" -#include "sizeclasstable.h" #include "ticker.h" diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index 0284e8a5d..e753f125c 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -2,7 +2,6 @@ #include "../ds/ds.h" #include "freelist.h" -#include "sizeclasstable.h" #include "snmalloc/stl/new.h" namespace snmalloc diff --git a/src/snmalloc/mem/remotecache.h b/src/snmalloc/mem/remotecache.h index 045ab141b..c92c99a65 100644 --- a/src/snmalloc/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -5,7 +5,6 @@ #include "freelist.h" #include "metadata.h" #include "remoteallocator.h" -#include "sizeclasstable.h" #include "snmalloc/stl/array.h" #include "snmalloc/stl/atomic.h" diff --git a/src/snmalloc/mem/secondary/default.h b/src/snmalloc/mem/secondary/default.h index ae5d53bfb..1fc86af46 100644 --- a/src/snmalloc/mem/secondary/default.h +++ b/src/snmalloc/mem/secondary/default.h @@ -1,7 +1,7 @@ #pragma once #include "snmalloc/ds_core/defines.h" -#include "snmalloc/ds_core/mitigations.h" +#include "snmalloc/mitigations/mitigations.h" #include diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index 178a3eeee..716c57935 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -1,8 +1,8 @@ #pragma once #include "gwp_asan/guarded_pool_allocator.h" +#include "snmalloc/ds/sizeclasstable.h" #include "snmalloc/ds_core/defines.h" -#include "snmalloc/mem/sizeclasstable.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif diff --git a/src/snmalloc/mitigations/allocconfig.h b/src/snmalloc/mitigations/allocconfig.h new file mode 100644 index 000000000..3f326a570 --- /dev/null +++ b/src/snmalloc/mitigations/allocconfig.h @@ -0,0 +1,102 @@ +#pragma once + +#include "../ds_core/sizeclassconfig.h" +#include "mitigations.h" + +namespace snmalloc +{ + // Minimum number of objects on a slab + static constexpr size_t MIN_OBJECT_COUNT = + mitigations(random_larger_thresholds) ? 13 : 4; + + /** + * The number of bits needed to count the number of objects within a slab. + * + * Most likely, this is achieved by the smallest sizeclass, which will have + * many more than MIN_OBJECT_COUNT objects in its slab. But, just in case, + * it's defined here and checked when we compute the sizeclass table, since + * computing this number is potentially nontrivial. + */ +#if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) + static constexpr size_t MAX_CAPACITY_BITS = 13; +#else + static constexpr size_t MAX_CAPACITY_BITS = 11; +#endif + + /** + * The maximum distance between the start of two objects in the same slab. + */ + static constexpr size_t MAX_SLAB_SPAN_SIZE = + (MIN_OBJECT_COUNT - 1) * MAX_SMALL_SIZECLASS_SIZE; + static constexpr size_t MAX_SLAB_SPAN_BITS = + bits::next_pow2_bits_const(MAX_SLAB_SPAN_SIZE); + + // Number of slots for remote deallocation. + static constexpr size_t REMOTE_SLOT_BITS = 8; + static constexpr size_t REMOTE_SLOTS = 1 << REMOTE_SLOT_BITS; + static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1; + +#if defined(SNMALLOC_DEALLOC_BATCH_RING_ASSOC) + static constexpr size_t DEALLOC_BATCH_RING_ASSOC = + SNMALLOC_DEALLOC_BATCH_RING_ASSOC; +#else +# if defined(__has_cpp_attribute) +# if ( \ + __has_cpp_attribute(msvc::no_unique_address) && \ + (__cplusplus >= 201803L || _MSVC_LANG >= 201803L)) || \ + __has_cpp_attribute(no_unique_address) + // For C++20 or later, we do have [[no_unique_address]] and so can also do + // batching if we aren't turning on the backward-pointer mitigations + static constexpr size_t DEALLOC_BATCH_MIN_ALLOC_WORDS = + mitigations(freelist_backward_edge) ? 4 : 2; +# else + // For C++17, we don't have [[no_unique_address]] and so we always end up + // needing all four pointers' worth of space (because BatchedRemoteMessage has + // two freelist::Object::T<> links within, each of which will have two fields + // and will be padded to two pointers). + static constexpr size_t DEALLOC_BATCH_MIN_ALLOC_WORDS = 4; +# endif +# else + // If we don't even have the feature test macro, we're C++17 or earlier. + static constexpr size_t DEALLOC_BATCH_MIN_ALLOC_WORDS = 4; +# endif + + static constexpr size_t DEALLOC_BATCH_RING_ASSOC = + (MIN_ALLOC_SIZE >= (DEALLOC_BATCH_MIN_ALLOC_WORDS * sizeof(void*))) ? 2 : 0; +#endif + +#if defined(SNMALLOC_DEALLOC_BATCH_RING_SET_BITS) + static constexpr size_t DEALLOC_BATCH_RING_SET_BITS = + SNMALLOC_DEALLOC_BATCH_RING_SET_BITS; +#else + static constexpr size_t DEALLOC_BATCH_RING_SET_BITS = 3; +#endif + + static constexpr size_t DEALLOC_BATCH_RINGS = + DEALLOC_BATCH_RING_ASSOC * bits::one_at_bit(DEALLOC_BATCH_RING_SET_BITS); + + // Return remote small allocs when the local cache reaches this size. + static constexpr int64_t REMOTE_CACHE = +#ifdef USE_REMOTE_CACHE + USE_REMOTE_CACHE +#else + MIN_CHUNK_SIZE +#endif + ; + + // Stop processing remote batch when we reach this amount of deallocations in + // bytes + static constexpr int64_t REMOTE_BATCH_LIMIT = +#ifdef SNMALLOC_REMOTE_BATCH_PROCESS_SIZE + SNMALLOC_REMOTE_BATCH_PROCESS_SIZE +#else + 1 * 1024 * 1024 +#endif + ; + + // Used to configure when the backend should use thread local buddies. + // This only basically is used to disable some buddy allocators on small + // fixed heap scenarios like OpenEnclave. + static constexpr size_t MIN_HEAP_SIZE_FOR_THREAD_LOCAL_BUDDY = + bits::one_at_bit(27); +} // namespace snmalloc diff --git a/src/snmalloc/ds_core/cheri.h b/src/snmalloc/mitigations/cheri.h similarity index 100% rename from src/snmalloc/ds_core/cheri.h rename to src/snmalloc/mitigations/cheri.h diff --git a/src/snmalloc/ds_core/mitigations.h b/src/snmalloc/mitigations/mitigations.h similarity index 99% rename from src/snmalloc/ds_core/mitigations.h rename to src/snmalloc/mitigations/mitigations.h index 4f5200853..5cd46911f 100644 --- a/src/snmalloc/ds_core/mitigations.h +++ b/src/snmalloc/mitigations/mitigations.h @@ -1,5 +1,5 @@ #pragma once -#include "defines.h" +#include "../ds_core/defines.h" #include diff --git a/src/snmalloc/mitigations/mitigations_all.h b/src/snmalloc/mitigations/mitigations_all.h new file mode 100644 index 000000000..379455642 --- /dev/null +++ b/src/snmalloc/mitigations/mitigations_all.h @@ -0,0 +1,12 @@ +#pragma once +/** + * The mitigations layer provides compile-time configuration for security + * mitigations. It sits between ds_aal/ and pal/ in the include hierarchy. + * + * Files in this directory may include ds_core/ and ds_aal/ but not pal/ + * or anything above. + */ + +#include "allocconfig.h" +#include "cheri.h" +#include "mitigations.h" diff --git a/src/snmalloc/pal/pal.h b/src/snmalloc/pal/pal.h index 6fbb8f347..884775459 100644 --- a/src/snmalloc/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -7,12 +7,14 @@ * may hide some of this or provide its own abstractions for sandboxing or * isolating heaps. * - * Files in this directory may depend on the architecture abstraction and core - * layers (`aal` and `ds_core`, respectively) but nothing else in snmalloc. + * Files in this directory may depend on the architecture abstraction, core, + * and mitigations layers (`aal`, `ds_core`, and `mitigations/` respectively) + * but nothing else in snmalloc. */ #pragma once #include "../aal/aal.h" +#include "../mitigations/mitigations_all.h" #include "pal_concept.h" #include "pal_consts.h" @@ -66,11 +68,17 @@ namespace snmalloc # error Unsupported platform #endif - [[noreturn]] SNMALLOC_SLOW_PATH inline void error(const char* const str) + [[noreturn]] SNMALLOC_SLOW_PATH SNMALLOC_USED_FUNCTION inline void + error(const char* const str) { DefaultPal::error(str); } + SNMALLOC_USED_FUNCTION inline void message_impl(const char* const str) + { + DefaultPal::message(str); + } + // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = DefaultPal::page_size; @@ -149,35 +157,4 @@ namespace snmalloc "Page size from system header does not match snmalloc config page size."); #endif - /** - * Report a fatal error via a PAL-specific error reporting mechanism. This - * takes a format string and a set of arguments. The format string indicates - * the remaining arguments with "{}". This could be extended later to - * support indexing fairly easily, if we ever want to localise these error - * messages. - * - * The following are supported as arguments: - * - * - Characters (`char`), printed verbatim. - * - Strings Literals (`const char*` or `const char[]`), printed verbatim. - * - Raw pointers (void*), printed as hex strings. - * - Integers (convertible to `size_t`), printed as hex strings. - * - * These types should be sufficient for allocator-related error messages. - */ - template - [[noreturn]] inline void report_fatal_error(Args... args) - { - MessageBuilder msg{stl::forward(args)...}; - DefaultPal::error(msg.get_message()); - } - - template - inline void message(Args... args) - { - MessageBuilder msg{stl::forward(args)...}; - MessageBuilder msg_tid{ - "{}: {}", debug_get_tid(), msg.get_message()}; - DefaultPal::message(msg_tid.get_message()); - } } // namespace snmalloc diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index 0046516a3..d1ffcf90e 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -3,8 +3,8 @@ */ #include -#include #include +#include void test_ctz() { diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index c28d381af..d375cbef0 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void alloc1(size_t size) @@ -47,14 +47,14 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r); } void calloc2(size_t size) { - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r, size); } @@ -109,7 +109,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::max_small_sizeclass_bits(); exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index b3ab963cd..3dcfb4559 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -1,12 +1,8 @@ +#include #include #include #include - -#define SNMALLOC_NAME_MANGLE(a) our_##a -#undef SNMALLOC_NO_REALLOCARRAY -#undef SNMALLOC_NO_REALLOCARR -#define SNMALLOC_BOOTSTRAP_ALLOCATOR -#include +#include using namespace snmalloc; @@ -27,8 +23,8 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) INFO("Unexpected null returned.\n"); failed = true; } - const auto alloc_size = our_malloc_usable_size(p); - auto expected_size = our_malloc_good_size(size); + const auto alloc_size = testlib_malloc_usable_size(p); + auto expected_size = testlib_malloc_good_size(size); const auto exact_size = align == 1; #ifdef __CHERI_PURE_CAPABILITY__ const auto cheri_size = __builtin_cheri_length_get(p); @@ -93,14 +89,14 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) } EXPECT(!failed, "check_result failed! {}", p); - our_free(p); + testlib_free(p); } void test_calloc(size_t nmemb, size_t size, int err, bool null) { START_TEST("calloc({}, {}) combined size {}\n", nmemb, size, nmemb * size); errno = SUCCESS; - void* p = our_calloc(nmemb, size); + void* p = testlib_calloc(nmemb, size); if (p != nullptr) { @@ -116,23 +112,23 @@ void test_realloc(void* p, size_t size, int err, bool null) { size_t old_size = 0; if (p != nullptr) - old_size = our_malloc_usable_size(p); + old_size = testlib_malloc_usable_size(p); START_TEST("realloc({}({}), {})", p, old_size, size); errno = SUCCESS; - auto new_p = our_realloc(p, size); + auto new_p = testlib_realloc(p, size); check_result(size, 1, new_p, err, null); // Realloc failure case, deallocate original block as not // handled by check_result. if (new_p == nullptr && size != 0) - our_free(p); + testlib_free(p); } void test_posix_memalign(size_t size, size_t align, int err, bool null) { START_TEST("posix_memalign(&p, {}, {})", align, size); void* p = nullptr; - errno = our_posix_memalign(&p, align, size); + errno = testlib_posix_memalign(&p, align, size); check_result(size, align, p, err, null); } @@ -140,7 +136,7 @@ void test_memalign(size_t size, size_t align, int err, bool null) { START_TEST("memalign({}, {})", align, size); errno = SUCCESS; - void* p = our_memalign(align, size); + void* p = testlib_memalign(align, size); check_result(size, align, p, err, null); } @@ -149,13 +145,13 @@ void test_reallocarray(void* p, size_t nmemb, size_t size, int err, bool null) size_t old_size = 0; size_t tsize = nmemb * size; if (p != nullptr) - old_size = our_malloc_usable_size(p); + old_size = testlib_malloc_usable_size(p); START_TEST("reallocarray({}({}), {})", p, old_size, tsize); errno = SUCCESS; - auto new_p = our_reallocarray(p, nmemb, size); + auto new_p = testlib_reallocarray(p, nmemb, size); if (new_p == nullptr && tsize != 0) - our_free(p); + testlib_free(p); check_result(tsize, 1, new_p, err, null); } @@ -165,29 +161,29 @@ void test_reallocarr( void* p = nullptr; if (size_old != (size_t)~0) - p = our_malloc(size_old); + p = testlib_malloc(size_old); START_TEST("reallocarr({}({}), {})", p, nmemb, size); errno = SUCCESS; - int r = our_reallocarr(&p, nmemb, size); + int r = testlib_reallocarr(&p, nmemb, size); EXPECT(r == err, "reallocarr failed! expected {} got {}\n", err, r); check_result(nmemb * size, 1, p, err, null); - p = our_malloc(size); + p = testlib_malloc(size); if (!p) { return; } for (size_t i = 1; i < size; i++) static_cast(p)[i] = 1; - our_reallocarr(&p, nmemb, size); + testlib_reallocarr(&p, nmemb, size); if (r != SUCCESS) - our_free(p); + testlib_free(p); for (size_t i = 1; i < size; i++) { EXPECT(static_cast(p)[i] == 1, "data consistency failed! at {}", i); } - our_free(p); + testlib_free(p); } int main(int argc, char** argv) @@ -223,31 +219,31 @@ int main(int argc, char** argv) abort(); } - our_free(nullptr); + testlib_free(nullptr); /* A very large allocation size that we expect to fail. */ const size_t too_big_size = ((size_t)-1) / 2; - check_result(too_big_size, 1, our_malloc(too_big_size), ENOMEM, true); + check_result(too_big_size, 1, testlib_malloc(too_big_size), ENOMEM, true); errno = SUCCESS; - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); START_TEST("malloc: {}", size); errno = SUCCESS; - check_result(size, 1, our_malloc(size), SUCCESS, false); + check_result(size, 1, testlib_malloc(size), SUCCESS, false); errno = SUCCESS; - check_result(size + 1, 1, our_malloc(size + 1), SUCCESS, false); + check_result(size + 1, 1, testlib_malloc(size + 1), SUCCESS, false); } test_calloc(0, 0, SUCCESS, false); - our_free(nullptr); + testlib_free(nullptr); test_calloc(1, too_big_size, ENOMEM, true); errno = SUCCESS; - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); @@ -268,38 +264,38 @@ int main(int argc, char** argv) // Check realloc(nullptr,0) behaves like malloc(1) test_realloc(nullptr, 0, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); - test_realloc(our_malloc(size), size, SUCCESS, false); + test_realloc(testlib_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < NUM_SMALL_SIZECLASSES; sc2++) + test_realloc(testlib_malloc(size), too_big_size, ENOMEM, true); + for (smallsizeclass_t sc2(0); sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); - test_realloc(our_malloc(size), size2, SUCCESS, false); - test_realloc(our_malloc(size + 1), size2, SUCCESS, false); + test_realloc(testlib_malloc(size), size2, SUCCESS, false); + test_realloc(testlib_malloc(size + 1), size2, SUCCESS, false); } // Check realloc(p,0), behaves like free(p), if p != nullptr - test_realloc(our_malloc(size), 0, SUCCESS, true); + test_realloc(testlib_malloc(size), 0, SUCCESS, true); } - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - test_realloc(our_malloc(size), size, SUCCESS, false); + test_realloc(testlib_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + test_realloc(testlib_malloc(size), too_big_size, ENOMEM, true); + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); INFO("size1: {}, size2:{}\n", size, size2); - test_realloc(our_malloc(size), size2, SUCCESS, false); - test_realloc(our_malloc(size + 1), size2, SUCCESS, false); + test_realloc(testlib_malloc(size), size2, SUCCESS, false); + test_realloc(testlib_malloc(size + 1), size2, SUCCESS, false); } } - test_realloc(our_malloc(64), 4194304, SUCCESS, false); + test_realloc(testlib_malloc(64), 4194304, SUCCESS, false); test_posix_memalign(0, 0, EINVAL, true); test_posix_memalign(too_big_size, 0, EINVAL, true); @@ -311,7 +307,7 @@ int main(int argc, char** argv) // Check overflow with alignment taking it round to 0. test_memalign(1 - align, align, ENOMEM, true); - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES - 6; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES - 6; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); @@ -324,37 +320,37 @@ int main(int argc, char** argv) } test_reallocarray(nullptr, 1, 0, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - test_reallocarray(our_malloc(size), 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, 0, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, size, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, 0, SUCCESS, false); test_reallocarray(nullptr, 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + test_reallocarray(testlib_malloc(size), 1, too_big_size, ENOMEM, true); + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); - test_reallocarray(our_malloc(size), 1, size2, SUCCESS, false); - test_reallocarray(our_malloc(size + 1), 1, size2, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, size2, SUCCESS, false); + test_reallocarray(testlib_malloc(size + 1), 1, size2, SUCCESS, false); } } test_reallocarr((size_t)~0, 1, 0, SUCCESS, false); test_reallocarr((size_t)~0, 1, 16, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_reallocarr(size, 1, size, SUCCESS, false); test_reallocarr(size, 1, 0, SUCCESS, false); test_reallocarr(size, 2, size, SUCCESS, false); - void* p = our_malloc(size); + void* p = testlib_malloc(size); EXPECT(p != nullptr, "realloc alloc failed with {}", size); - int r = our_reallocarr(&p, 1, too_big_size); + int r = testlib_reallocarr(&p, 1, too_big_size); EXPECT(r == ENOMEM, "expected failure on allocation\n"); - our_free(p); + testlib_free(p); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); START_TEST("size1: {}, size2:{}", size, size2); @@ -363,7 +359,7 @@ int main(int argc, char** argv) } EXPECT( - our_malloc_usable_size(nullptr) == 0, + testlib_malloc_usable_size(nullptr) == 0, "malloc_usable_size(nullptr) should be zero"); snmalloc::debug_check_empty(); diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 7c181a24a..9bf335087 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,9 +1,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -178,7 +178,7 @@ void test_calloc() memset(p, 0xFF, size); snmalloc::dealloc(p, size); - p = snmalloc::alloc(size); + p = snmalloc::alloc(size); for (size_t i = 0; i < size; i++) { @@ -239,11 +239,11 @@ void test_double_alloc() void test_external_pointer() { - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass_const(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { - size_t size = sizeclass_to_size(sc); + size_t size = sizeclass_to_size_const(sc); void* p1 = snmalloc::alloc(size); if (size != snmalloc::alloc_size(p1)) @@ -259,8 +259,8 @@ void test_external_pointer() for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); - void* p3 = snmalloc::external_pointer(p2); - void* p4 = snmalloc::external_pointer(p2); + void* p3 = snmalloc::libc::__malloc_start_pointer(p2); + void* p4 = snmalloc::libc::__malloc_last_byte_pointer(p2); if (p1 != p3) { if (p3 > p1 || snmalloc::is_owned(p1)) @@ -293,7 +293,7 @@ void test_external_pointer() void check_offset(void* base, void* interior) { - void* calced_base = snmalloc::external_pointer((void*)interior); + void* calced_base = snmalloc::libc::__malloc_start_pointer((void*)interior); if (calced_base != (void*)base) { if (calced_base > base || snmalloc::is_owned(base)) @@ -320,10 +320,10 @@ void test_external_pointer_large() { xoroshiro::p128r64 r; - constexpr size_t count_log = DefaultPal::address_bits > 32 ? 5 : 3; - constexpr size_t count = 1 << count_log; + const size_t count_log = pal_address_bits() > 32 ? 5 : 3; + const size_t count = size_t(1) << count_log; // Pre allocate all the objects - size_t* objects[count]; + size_t* objects[1 << 5]; // max possible count size_t total_size = 0; @@ -376,7 +376,7 @@ void test_external_pointer_dealloc_bug() for (size_t i = 0; i < count; i++) { - snmalloc::external_pointer(allocs[i]); + snmalloc::libc::__malloc_start_pointer(allocs[i]); } snmalloc::dealloc(allocs[0]); @@ -391,10 +391,11 @@ void test_external_pointer_stack() for (size_t i = 0; i < stack.size(); i++) { - if (snmalloc::external_pointer(&stack[i]) > &stack[i]) + if (snmalloc::libc::__malloc_start_pointer(&stack[i]) > &stack[i]) { std::cout << "Stack pointer: " << &stack[i] << " external pointer: " - << snmalloc::external_pointer(&stack[i]) << std::endl; + << snmalloc::libc::__malloc_start_pointer(&stack[i]) + << std::endl; abort(); } } @@ -408,7 +409,8 @@ void test_alloc_16M() const size_t size = 16'000'000; void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -417,8 +419,9 @@ void test_calloc_16M() // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -430,8 +433,9 @@ void test_calloc_large_bug() // not a multiple of page size. const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; - void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -485,16 +489,15 @@ void test_static_sized_allocs() void test_remaining_bytes() { - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass_const(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { - auto size = sizeclass_to_size(sc); + auto size = sizeclass_to_size_const(sc); char* p = (char*)snmalloc::alloc(size); for (size_t offset = 0; offset < size; offset++) { - auto rem = - snmalloc::remaining_bytes(address_cast(pointer_offset(p, offset))); + auto rem = snmalloc::remaining_bytes(pointer_offset(p, offset)); if (rem != (size - offset)) { if (rem < (size - offset) || snmalloc::is_owned(p)) diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 68fa179f7..036bcfe2f 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -4,12 +4,9 @@ */ #include #include +#include #include -#define SNMALLOC_NAME_MANGLE(a) our_##a -#include "../../../snmalloc/override/malloc-extensions.cc" -#include "../../../snmalloc/override/malloc.cc" - using namespace snmalloc; bool print_memory_usage() @@ -43,7 +40,7 @@ void add_n_allocs(size_t n) { while (true) { - auto p = our_malloc(1024); + auto p = testlib_malloc(1024); allocs.push_back(p); if (print_memory_usage()) { @@ -64,7 +61,7 @@ void remove_n_allocs(size_t n) if (allocs.empty()) return; auto p = allocs.back(); - our_free(p); + testlib_free(p); allocs.pop_back(); if (print_memory_usage()) { diff --git a/src/test/func/multi_atexit/multi_atexit.cc b/src/test/func/multi_atexit/multi_atexit.cc index e4f5ed327..09919ada3 100644 --- a/src/test/func/multi_atexit/multi_atexit.cc +++ b/src/test/func/multi_atexit/multi_atexit.cc @@ -13,8 +13,8 @@ #endif #ifdef RUN_TEST -# include # include +# include void do_nothing() {} diff --git a/src/test/func/multi_threadatexit/multi_threadatexit.cc b/src/test/func/multi_threadatexit/multi_threadatexit.cc index 85f918f93..8e4ce1de3 100644 --- a/src/test/func/multi_threadatexit/multi_threadatexit.cc +++ b/src/test/func/multi_threadatexit/multi_threadatexit.cc @@ -15,8 +15,8 @@ #endif #ifdef RUN_TEST -# include # include +# include template void thread_destruct() diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index a0b53689f..7a03fa1a7 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -7,8 +7,9 @@ */ #include -#include +#include #include +#include using namespace snmalloc; static constexpr size_t GRANULARITY_BITS = 20; diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 6a2874215..8f11ff689 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -1,8 +1,8 @@ #include #include -#include -#include +#include #include +#include #include using namespace snmalloc; @@ -56,7 +56,7 @@ void test_alloc() auto ptr = PoolA::acquire(); SNMALLOC_CHECK(ptr != nullptr); // Pool allocations should not be visible to debug_check_empty. - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); PoolA::release(ptr); } @@ -242,13 +242,7 @@ void test_sort() int main(int argc, char** argv) { setup(); -#ifdef USE_SYSTEMATIC_TESTING - opt::Opt opt(argc, argv); - size_t seed = opt.is("--seed", 0); - Virtual::systematic_bump_ptr() += seed << 17; -#else UNUSED(argc, argv); -#endif test_double_alloc(); std::cout << "test_double_alloc passed" << std::endl; diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index 77227b202..aeade74f1 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -9,8 +9,10 @@ int main() # define SNMALLOC_PTHREAD_FORK_PROTECTION # include -# include +# include +# include # include +# include void simulate_allocation() { diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 164a5978f..61fccb6d3 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -11,7 +11,9 @@ # define SNMALLOC_TRACING #endif // Redblack tree needs some libraries with trace enabled. -#include "snmalloc/snmalloc.h" +#include "test/snmalloc_testlib.h" + +#include struct NodeRef { diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index f9541331c..4d11eaafb 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -1,6 +1,7 @@ #include -#include +#include #include +#include using namespace snmalloc; // Check for all sizeclass that we correctly round every offset within @@ -17,9 +18,10 @@ int main(int argc, char** argv) bool failed = false; - for (size_t size_class = 0; size_class < NUM_SMALL_SIZECLASSES; size_class++) + for (smallsizeclass_t size_class; size_class < NUM_SMALL_SIZECLASSES; + size_class++) { - size_t rsize = sizeclass_to_size((uint8_t)size_class); + size_t rsize = sizeclass_to_size(size_class); size_t max_offset = sizeclass_to_slab_size(size_class); sizeclass_t sc = sizeclass_t::from_small_class(size_class); for (size_t offset = 0; offset < max_offset; offset++) diff --git a/src/test/func/seqset/seqset.cc b/src/test/func/seqset/seqset.cc index 36737f29f..429d8cd7a 100644 --- a/src/test/func/seqset/seqset.cc +++ b/src/test/func/seqset/seqset.cc @@ -10,7 +10,9 @@ * The corruption case is run in a forked child so that the expected abort * does not kill the test harness itself. */ -#include +#include +#include +#include #include #include diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 836c62111..ac7ec6bd8 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -1,6 +1,7 @@ #include -#include +#include #include +#include NOINLINE snmalloc::smallsizeclass_t size_to_sizeclass(size_t size) @@ -17,9 +18,7 @@ void test_align_size() SNMALLOC_CHECK(snmalloc::aligned_size(128, 160) == 256); - for (size_t size = 1; - size < snmalloc::sizeclass_to_size(snmalloc::NUM_SMALL_SIZECLASSES - 1); - size++) + for (size_t size = 1; snmalloc::is_small_sizeclass(size); size++) { size_t rsize = snmalloc::round_size(size); @@ -85,7 +84,7 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::smallsizeclass_t sz = 0; sz < snmalloc::NUM_SMALL_SIZECLASSES; + for (snmalloc::smallsizeclass_t sz(0); sz < snmalloc::NUM_SMALL_SIZECLASSES; sz++) { if ( diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index d66f060a1..cf3596748 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -1,5 +1,5 @@ #include -#include +#include #include template diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index b20d8b1cf..02129374d 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void trigger_teardown() @@ -59,7 +59,7 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { trigger_teardown(); - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r); } @@ -67,7 +67,7 @@ void calloc1(size_t size) void calloc2(size_t size) { trigger_teardown(); - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r, size); } @@ -124,7 +124,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::max_small_sizeclass_bits(); exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/helpers.h b/src/test/helpers.h index 30f6e4655..d8738ddc9 100644 --- a/src/test/helpers.h +++ b/src/test/helpers.h @@ -18,8 +18,7 @@ namespace snmalloc do \ { \ current_test = __PRETTY_FUNCTION__; \ - MessageBuilder<1024> mb{"Starting test: " msg "\n", ##__VA_ARGS__}; \ - DefaultPal::message(mb.get_message()); \ + snmalloc::message<1024>("Starting test: " msg, ##__VA_ARGS__); \ } while (0) /** @@ -32,8 +31,7 @@ namespace snmalloc #define INFO(msg, ...) \ do \ { \ - MessageBuilder<1024> mb{msg "\n", ##__VA_ARGS__}; \ - DefaultPal::message(mb.get_message()); \ + snmalloc::message<1024>(msg, ##__VA_ARGS__); \ } while (0) } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 6bed00545..bca2a4889 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -5,8 +5,7 @@ #include #include -#include -#include +#include #include #include @@ -31,18 +30,18 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = DefaultPal::tick(); + start = snmalloc::pal_tick(); flag = true; } while (!flag) - Aal::pause(); + snmalloc::pal_pause(); f(id); prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = DefaultPal::tick(); + end = snmalloc::pal_tick(); } } diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 76be4c708..07e69cef9 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -1,6 +1,6 @@ -#include #include #include +#include #include #include @@ -19,7 +19,7 @@ namespace test { size_t rand = (size_t)r.next(); size_t offset = bits::clz(rand); - if constexpr (DefaultPal::address_bits > 32) + if (snmalloc::pal_address_bits() > 32) { if (offset > 30) offset = 30; @@ -80,7 +80,8 @@ namespace test size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); - void* calced_external = snmalloc::external_pointer(interior_ptr); + void* calced_external = + snmalloc::libc::__malloc_start_pointer(interior_ptr); if (calced_external != external_ptr) { abort(); diff --git a/src/test/perf/large_alloc/large_alloc.cc b/src/test/perf/large_alloc/large_alloc.cc index 2532b41e3..b0f0f2bc8 100644 --- a/src/test/perf/large_alloc/large_alloc.cc +++ b/src/test/perf/large_alloc/large_alloc.cc @@ -1,6 +1,6 @@ -#include #include #include +#include using namespace snmalloc; diff --git a/src/test/perf/lotsofthreads/lotsofthread.cc b/src/test/perf/lotsofthreads/lotsofthread.cc index 000af6534..9705dfff3 100644 --- a/src/test/perf/lotsofthreads/lotsofthread.cc +++ b/src/test/perf/lotsofthreads/lotsofthread.cc @@ -19,7 +19,7 @@ #include using namespace std; -#include +#include #define malloc snmalloc::libc::malloc #define free snmalloc::libc::free #define malloc_usable_size snmalloc::libc::malloc_usable_size diff --git a/src/test/perf/post_teardown/post-teardown.cc b/src/test/perf/post_teardown/post-teardown.cc index ca9296d71..915513c08 100644 --- a/src/test/perf/post_teardown/post-teardown.cc +++ b/src/test/perf/post_teardown/post-teardown.cc @@ -1,7 +1,7 @@ #include -#include #include #include +#include #include using namespace snmalloc; @@ -11,7 +11,7 @@ void fill(std::vector& out, size_t count, size_t size) out.reserve(count); for (size_t i = 0; i < count; i++) { - out.push_back(snmalloc::alloc(size)); + out.push_back(snmalloc::alloc(size)); } } @@ -41,7 +41,7 @@ int main(int, char**) // Simulate the allocator already being torn down before remaining frees // (post-main / static destruction path from #809). - ThreadAlloc::teardown(); + debug_teardown(); fill(ptrs, alloc_count, obj_size); drain("Immediate dealloc after teardown", ptrs, obj_size); diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index ed068a31f..bf173969d 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -1,25 +1,25 @@ -#include #include #include +#include #include using namespace snmalloc; -template +template void test_alloc_dealloc(size_t count, size_t size, bool write) { { MeasureTime m; m << "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) - << size - << ", ZeroMem: " << std::is_same_v << ", Write: " << write; + << size << ", ZeroMem: " << (zero_mem == ZeroMem::YesZero) + << ", Write: " << write; std::unordered_set set; // alloc 1.5x objects for (size_t i = 0; i < ((count * 3) / 2); i++) { - void* p = snmalloc::alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -41,7 +41,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) // alloc 1x objects for (size_t i = 0; i < count; i++) { - void* p = snmalloc::alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -68,18 +68,18 @@ int main(int, char**) for (size_t size = 16; size <= 128; size <<= 1) { - test_alloc_dealloc(1 << 15, size, false); - test_alloc_dealloc(1 << 15, size, true); - test_alloc_dealloc(1 << 15, size, false); - test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); } for (size_t size = 1 << 12; size <= 1 << 17; size <<= 1) { - test_alloc_dealloc(1 << 10, size, false); - test_alloc_dealloc(1 << 10, size, true); - test_alloc_dealloc(1 << 10, size, false); - test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); } return 0; diff --git a/src/test/perf/startup/startup.cc b/src/test/perf/startup/startup.cc index dfc6cb0c2..30fa23438 100644 --- a/src/test/perf/startup/startup.cc +++ b/src/test/perf/startup/startup.cc @@ -5,8 +5,7 @@ #include #include -#include -#include +#include #include #include @@ -31,18 +30,18 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = DefaultPal::tick(); + start = snmalloc::pal_tick(); flag = true; } while (!flag) - Aal::pause(); + snmalloc::pal_pause(); f(id); prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = DefaultPal::tick(); + end = snmalloc::pal_tick(); } } @@ -77,9 +76,9 @@ int main() ParallelTest test( [](size_t id) { - auto start = DefaultPal::tick(); + auto start = snmalloc::pal_tick(); snmalloc::dealloc(snmalloc::alloc(1)); - auto end = DefaultPal::tick(); + auto end = snmalloc::pal_tick(); counters[id] = end - start; }, nthreads); diff --git a/src/test/snmalloc_testlib.cc b/src/test/snmalloc_testlib.cc new file mode 100644 index 000000000..b9dce251e --- /dev/null +++ b/src/test/snmalloc_testlib.cc @@ -0,0 +1,196 @@ +/** + * @file snmalloc_testlib.cc + * @brief Single translation unit that compiles the full snmalloc allocator + * and provides symbols for test-library consumers. + * + * Built once per flavour (fast / check) into a static library that tests + * link against, avoiding redundant recompilation of the allocator in every + * test TU. + * + * Three kinds of symbols are provided: + * + * A. Non-templated inline functions (dealloc, debug_teardown, libc::*) + * are marked SNMALLOC_USED_FUNCTION in their declarations, so weak/COMDAT + * symbols are emitted in every TU that includes them; the linker resolves + * test references to the copies in this archive. + * + * B. Templates on user-visible params only (alloc): + * explicit instantiation definitions force emission of strong symbols. + * + * C. Config-templated functions: non-template overloads that call through + * to the template version. These are distinct functions (not + * ODR-conflicting with the templates). + */ + +// On MSVC (where SNMALLOC_USED_FUNCTION is empty), override the testlib +// inline macros to produce strong (non-inline) definitions so the linker +// can resolve ARM64EC exit-thunks. On GCC/Clang, __attribute__((used)) +// already handles this. +#ifdef _MSC_VER +# define SNMALLOC_API NOINLINE +# define SNMALLOC_API_SLOW NOINLINE +#endif + +#include + +namespace snmalloc +{ + template + void* alloc(size_t size) + { + if constexpr (zero_mem == ZeroMem::YesZero) + { + return alloc(size); + } + else + { + return alloc(size); + } + } + + template void* alloc(size_t size); + template void* alloc(size_t size); + + // -- C: Non-template wrappers for Config-templated functions ------------- + + size_t alloc_size(const void* p) + { + return alloc_size(p); + } + + size_t remaining_bytes(const void* p) + { + return remaining_bytes(p); + } + + bool is_owned(void* p) + { + return is_owned(p); + } + + void debug_check_empty(bool* result) + { + debug_check_empty(result); + } + + void debug_in_use(size_t count) + { + debug_in_use(count); + } + + void cleanup_unused() + { + cleanup_unused(); + } + + // -- Opaque scoped allocator (inherits from ScopedAllocator<>) ----------- + + struct TestScopedAllocator : public ScopedAllocator<> + {}; + + TestScopedAllocator* create_scoped_allocator() + { + return new TestScopedAllocator(); + } + + void destroy_scoped_allocator(TestScopedAllocator* p) + { + delete p; + } + + void* scoped_alloc(TestScopedAllocator* a, size_t size) + { + return a->alloc->alloc(size); + } + + // ScopedAllocHandle is declared in snmalloc_testlib.h; define methods here. + struct ScopedAllocHandle + { + TestScopedAllocator* ptr; + ScopedAllocHandle(); + ~ScopedAllocHandle(); + ScopedAllocHandle(const ScopedAllocHandle&) = delete; + ScopedAllocHandle& operator=(const ScopedAllocHandle&) = delete; + void* alloc(size_t size); + void dealloc(void* p); + void dealloc(void* p, size_t size); + ScopedAllocHandle* operator->(); + const ScopedAllocHandle* operator->() const; + }; + + ScopedAllocHandle::ScopedAllocHandle() : ptr(create_scoped_allocator()) {} + + ScopedAllocHandle::~ScopedAllocHandle() + { + destroy_scoped_allocator(ptr); + } + + void* ScopedAllocHandle::alloc(size_t size) + { + return scoped_alloc(ptr, size); + } + + void ScopedAllocHandle::dealloc(void* p) + { + snmalloc::dealloc(p); + } + + void ScopedAllocHandle::dealloc(void* p, size_t size) + { + snmalloc::dealloc(p, size); + } + + ScopedAllocHandle* ScopedAllocHandle::operator->() + { + return this; + } + + const ScopedAllocHandle* ScopedAllocHandle::operator->() const + { + return this; + } + + ScopedAllocHandle get_scoped_allocator() + { + return {}; + } + + size_t max_small_sizeclass_bits() + { + return MAX_SMALL_SIZECLASS_BITS; + } + + // -- Explicit instantiations of libc::memcpy variants -------------------- + namespace libc + { + template void* memcpy(void*, const void*, size_t); + template void* memcpy(void*, const void*, size_t); + template void* memcpy(void*, const void*, size_t); + } // namespace libc + + // -- PAL/AAL wrappers ---------------------------------------------------- + + size_t pal_address_bits() + { + return DefaultPal::address_bits; + } + + uint64_t pal_tick() + { + return DefaultPal::tick(); + } + + void pal_pause() + { + Aal::pause(); + } +} // namespace snmalloc + +// -- override/malloc.cc symbols with testlib_ prefix ----------------------- +#undef SNMALLOC_NO_REALLOCARRAY +#undef SNMALLOC_NO_REALLOCARR +#define SNMALLOC_NAME_MANGLE(a) testlib_##a +#include + +// malloc-extensions (unprefixed - get_malloc_info_v1) +#include diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h new file mode 100644 index 000000000..343362999 --- /dev/null +++ b/src/test/snmalloc_testlib.h @@ -0,0 +1,182 @@ +#pragma once +/** + * @file snmalloc_testlib.h + * @brief Thin test-library header that replaces for + * tests that can be linked against a pre-compiled static library. + * + * Does NOT include snmalloc_core.h — only forward-declares the minimal types + * needed for the API surface. Tests that need snmalloc_core types (bits::*, + * sizeclass, Pal, etc.) should include themselves + * before this header. + * + * Config-templated functions are provided as non-template overloads + * (distinct from the templates, no ODR conflict) defined in testlib.cc. + */ + +#include +#include +#include +#include +#include +#include + +namespace snmalloc +{ + template + class DefaultConts; + // Uninit / Zero are usable as template arguments even without the full + // definition of DefaultConts (only needed inside the allocator itself). + using Uninit = DefaultConts; + using Zero = DefaultConts; + + enum Boundary + { + Start, + End, + OnePastEnd + }; + + // -- Non-template functions (symbol forced in testlib.cc) ---------------- + void dealloc(void* p); + void dealloc(void* p, size_t size); + void dealloc(void* p, size_t size, size_t align); + + template + inline void dealloc(void* p) + { + dealloc(p, size); + } + + void debug_teardown(); + + // -- Non-template wrappers for Config-templated functions ---------------- + size_t alloc_size(const void* p); + size_t remaining_bytes(const void* p); + bool is_owned(void* p); + void debug_check_empty(bool* result = nullptr); + void debug_in_use(size_t count); + void cleanup_unused(); + + // -- Opaque scoped allocator --------------------------------------------- + // TestScopedAllocator inherits from ScopedAllocator<> in testlib.cc. + // Forward-declared here; usable through ScopedAllocHandle. + struct TestScopedAllocator; + TestScopedAllocator* create_scoped_allocator(); + void destroy_scoped_allocator(TestScopedAllocator*); + void* scoped_alloc(TestScopedAllocator*, size_t size); + + struct ScopedAllocHandle + { + TestScopedAllocator* ptr; + + ScopedAllocHandle(); + ~ScopedAllocHandle(); + ScopedAllocHandle(const ScopedAllocHandle&) = delete; + ScopedAllocHandle& operator=(const ScopedAllocHandle&) = delete; + void* alloc(size_t size); + + void dealloc(void* p); + void dealloc(void* p, size_t size); + + ScopedAllocHandle* operator->(); + const ScopedAllocHandle* operator->() const; + }; + + ScopedAllocHandle get_scoped_allocator(); + + // -- Constants exposed from allocconfig.h -------------------------------- + size_t max_small_sizeclass_bits(); + + // -- PAL/AAL wrappers (avoids needing pal.h in tests) -------------------- + size_t pal_address_bits(); + uint64_t pal_tick(); + void pal_pause(); + + // Simple pointer arithmetic (matches snmalloc::pointer_offset for void*). + // Defined inline so tests that don't include aal/address.h can still use it. + inline void* pointer_offset(void* base, size_t diff) + { + return static_cast(static_cast(base) + diff); + } + + template + void* alloc(size_t size); + extern template void* alloc(size_t); + extern template void* alloc(size_t); + + template + void* alloc_aligned(size_t align, size_t size); + + // -- libc namespace ------------------------------------------------------ + namespace libc + { + void* malloc(size_t size); + void* malloc_small(smallsizeclass_t sizeclass); + void* malloc_small_zero(smallsizeclass_t sizeclass); + void free(void* ptr); + void* calloc(size_t nmemb, size_t size); + void* realloc(void* ptr, size_t size); + size_t malloc_usable_size(const void* ptr); + void* memalign(size_t alignment, size_t size); + void* aligned_alloc(size_t alignment, size_t size); + int posix_memalign(void** memptr, size_t alignment, size_t size); + void* reallocarray(void* ptr, size_t nmemb, size_t size); + int reallocarr(void* ptr_, size_t nmemb, size_t size); + void* __malloc_start_pointer(void* ptr); + void* __malloc_last_byte_pointer(void* ptr); + void* __malloc_end_pointer(void* ptr); + + template + void* memcpy(void* dst, const void* src, size_t len); + } // namespace libc + + /** + * Compile-time size alloc. When the size (after alignment) is a small + * sizeclass the sizeclass is computed at compile time and the allocation + * goes straight to the sizeclass-based fast path. Otherwise falls back + * to the dynamic alloc. + */ + template + inline void* alloc() + { + if constexpr (is_small_sizeclass(size)) + { + constexpr auto sc = size_to_sizeclass_const(size); + if constexpr (zero_mem == ZeroMem::YesZero) + { + return libc::malloc_small_zero(sc); + } + else + { + return libc::malloc_small(sc); + } + } + else + { + return alloc(size); + } + } +} // namespace snmalloc + +// -- malloc-extensions struct and function ---------------------------------- +#include + +// -- override/malloc.cc functions with testlib_ prefix ---------------------- +#ifndef MALLOC_USABLE_SIZE_QUALIFIER +# define MALLOC_USABLE_SIZE_QUALIFIER +#endif +extern "C" +{ + void* testlib_malloc(size_t size); + void testlib_free(void* ptr); + void testlib_cfree(void* ptr); + void* testlib_calloc(size_t nmemb, size_t size); + size_t testlib_malloc_usable_size(MALLOC_USABLE_SIZE_QUALIFIER void* ptr); + size_t testlib_malloc_good_size(size_t size); + void* testlib_realloc(void* ptr, size_t size); + void* testlib_reallocarray(void* ptr, size_t nmemb, size_t size); + int testlib_reallocarr(void* ptr, size_t nmemb, size_t size); + void* testlib_memalign(size_t alignment, size_t size); + void* testlib_aligned_alloc(size_t alignment, size_t size); + int testlib_posix_memalign(void** memptr, size_t alignment, size_t size); +}