diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a65308a6c..448a7ff3b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.5.1 + RELEASE: Release v3.5.2 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,14 +19,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.5.1, v2.5.1, v1.15.1] # [dev,dev2,dev3] + branch: [v3.5.2, v2.5.2, v1.15.2] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.branch }} fetch-tags: true @@ -45,7 +45,7 @@ jobs: # Create a release: https://github.com/softprops/action-gh-release (MIT license) - name: Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 if: ${{ env.RELEASE != '' && env.RELEASE != 'no' }} with: name: ${{ env.RELEASE }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 71a91a887..7079b90d0 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -11,7 +11,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: days-before-issue-stale: 360 days-before-issue-close: 14 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d95821599..8640938ec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,7 +55,7 @@ jobs: steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Choose MI_USE_CXX setting - name: Set default cmake options @@ -77,7 +77,7 @@ jobs: ctest --test-dir out/release-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Release, Asm - if: matrix.tests == 'basic' && runner.os == 'Linux' + if: matrix.tests == 'basic' && runner.os == 'Linux' && matrix.c_or_cxx == 'c' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-${{ matrix.c_or_cxx }}-asm @@ -238,20 +238,20 @@ jobs: - name: Debug, Guarded if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON + cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=FULL cmake --build out/debug-guarded --parallel 8 --config Debug ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug env: - MIMALLOC_GUARDED_SAMPLE_RATE: 10 + MIMALLOC_GUARDED_SAMPLE_RATE: 1024 - name: Release, Guarded if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') run: | - cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON + cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=FULL cmake --build out/release-guarded --parallel 8 --config Release ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release env: - MIMALLOC_GUARDED_SAMPLE_RATE: 10 + MIMALLOC_GUARDED_SAMPLE_RATE: 128 - name: Debug, Use pagemap if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') @@ -314,7 +314,7 @@ jobs: # VM: FreeBSD (arm64 and x64) - name: FreeBSD arm64, Setup if: matrix.tests == 'freebsd-arm64' - uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1.5.3 + uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1.5.5 with: arch: arm64 custom-shell-name: freebsd-arm64 @@ -353,7 +353,7 @@ jobs: - name: FreeBSD x64, Setup if: matrix.tests == 'freebsd-x64' - uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1.5.3 + uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1.5.5 with: arch: x86_64 usesh: true @@ -509,10 +509,17 @@ jobs: if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} run: | - cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON cmake --build out/release-arm32 --parallel 4 --config Release ctest --test-dir out/release-arm32 --verbose --timeout 240 -C Release + - name: Alpine arm32, Asm + if: matrix.tests == 'alpine-arm32' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: alpine-arm32-c-asm + path: out/release-arm32/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine arm32, Secure if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} diff --git a/CMakeLists.txt b/CMakeLists.txt index c47a3d05a..c8942709f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,9 @@ set_property(CACHE MI_SECURE PROPERTY STRINGS "OFF" "ON" "FULL") option(MI_OVERRIDE "Override the standard malloc interface (i.e. define entry points for 'malloc', 'free', etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) -option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) + +set(MI_GUARDED "DEFAULT" CACHE STRING "Build with ability to put guard pages behind certain object allocations") +set_property(CACHE MI_GUARDED PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") @@ -18,20 +20,26 @@ set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))") set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "avx2" "armv8.1" "armv8.3" "rva22" "rva23" "rv64gcb_zacas" "rv64gcb_zacas_zalasr" "DEFAULT") -option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) - -set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") -set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFAULT") - set(MI_DEBUG "DEFAULT" CACHE STRING "Enable assertion checks: OFF, ON, INTERNAL, FULL, DEFAULT. INTERNAL adds extra internal invariant checks, while FULL adds expensive heap invariant checks") set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAULT") set(MI_TRACK "OFF" CACHE STRING "Enable tracking: OFF, ASAN (address sanitizer), VALGRIND, ETW (Windows event tracing). Adds a small overhead.") set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") +set(MI_STATS "DEFAULT" CACHE STRING "Enable statistics.") +set_property(CACHE MI_STATS PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +set(MI_PROFILE "DEFAULT" CACHE STRING "Enable profiling.") +set_property(CACHE MI_PROFILE PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") +set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFAULT") + set(MI_ALLOW_THP "DEFAULT" CACHE STRING "Enable transparent huge pages on Linux: OFF, ON, FULL, DEFAULT. 'FULL' enables THP and also avoids splitting up huge pages during purging.") set_property(CACHE MI_ALLOW_THP PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") +option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) + option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) @@ -72,7 +80,6 @@ set_property(CACHE MI_TLS_MODEL PROPERTY STRINGS "LOCAL" "PTHREADS" "LOCAL_DYNAM # deprecated options option(MI_USE_LIBATOMIC "Deprecated: Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) -option(MI_OPT_FREE_SMALL "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) option(MI_DEBUG_INTERNAL "Deprecated: Enable assertion and internal invariant checks (use MI_DEBUG=INTERNAL instead)" OFF) option(MI_DEBUG_FULL "Deprecated: Enable assertion checks and expensive internal heap invariant checking (use MI_DEBUG=FULL instead)" OFF) option(MI_CHECK_FULL "Deprecated: Use full internal invariant checking in DEBUG mode (use MI_DEBUG=FULL instead)" OFF) @@ -89,6 +96,8 @@ option(MI_TRACK_VALGRIND "Deprecated: Compile with Valgrind support (adds a option(MI_TRACK_ASAN "Deprecated: Compile with address sanitizer support (adds a small overhead) (use MI_TRACK=ASAN instead)" OFF) option(MI_TRACK_ETW "Deprecated: Compile with Windows event tracing (ETW) support (adds a small overhead) (use MI_TRACK=ETW instead)" OFF) +set(MI_OPT_FREE_SMALL "DEFAULT" CACHE STRING "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)") +set_property(CACHE MI_OPT_FREE_SMALL PROPERTY STRINGS "OFF" "ON" "DEFAULT") include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) @@ -107,14 +116,16 @@ set(mi_sources src/options.c src/os.c src/page.c - src/page-map.c + src/page-map.c + src/prim/prim.c + src/prim/prim-tls.c src/random.c + src/sample-guarded.c + src/sample-profile.c src/stats.c src/subproc.c src/theap.c - src/threadlocal.c - src/prim/prim.c - src/prim/prim-tls.c) + src/threadlocal.c) set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build @@ -156,7 +167,7 @@ endfunction() # ----------------------------------------------------------------------------- message(STATUS "") -if(NOT CMAKE_BUILD_TYPE) +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) if ("${CMAKE_BINARY_DIR}" MATCHES ".*((D|d)ebug|asan|tsan|ubsan|valgrind)$") message(STATUS "No build type selected, default to 'Debug'") set(CMAKE_BUILD_TYPE "Debug") @@ -241,7 +252,7 @@ if(MI_USE_CXX STREQUAL "DEFAULT") if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) # force C++ compilation with msvc or clang-cl to use modern C++ atomics set(MI_USE_CXX "ON") - elseif(MI_DEBUG_UBSAN AND CMAKE_BUILD_TYPE MATCHES "Debug") # ubsan needs C++ + elseif(MI_DEBUG_UBSAN AND (CMAKE_BUILD_TYPE MATCHES "Debug" OR NOT CMAKE_CONFIGURATION_TYPES)) # ubsan needs C++ set(MI_USE_CXX "ON") else() set(MI_USE_CXX "OFF") @@ -410,10 +421,16 @@ elseif(MI_DEBUG_INTERNAL) endif() if(MI_DEBUG STREQUAL "DEFAULT") - if(CMAKE_BUILD_TYPE MATCHES "Debug") - set(MI_DEBUG "INTERNAL") + if(CMAKE_CONFIGURATION_TYPES) + # Multi-config generators: Choose debug flags using generator expression + list(APPEND mi_defines $,MI_DEBUG=2,MI_DEBUG=0 NDEBUG=1>) + set(MI_DEBUG OFF) # don't go into further MI_DEBUG checks below... else() - set(MI_DEBUG "OFF") + if(CMAKE_BUILD_TYPE MATCHES "Debug") + set(MI_DEBUG "INTERNAL") + else() + set(MI_DEBUG "OFF") + endif() endif() endif() @@ -430,15 +447,28 @@ elseif(MI_DEBUG STREQUAL "INTERNAL") elseif(MI_DEBUG) message(STATUS "Set debug level to assertion checking (MI_DEBUG=${MI_DEBUG})") list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) +elseif(NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Disable assertion checks (MI_DEBUG=${MI_DEBUG})") + list(APPEND mi_defines NDEBUG=1) endif() -if(MI_DEBUG AND !MI_GUARDED) - message(STATUS "Enable MI_GUARDED (since MI_DEBUG is enabled)") - set(MI_GUARDED=ON) +if(MI_GUARDED STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_GUARDED "FULL") + else() + set(MI_GUARDED "OFF") + endif() endif() -if(MI_GUARDED) - message(STATUS "Compile guard pages behind certain object allocations (MI_GUARDED=ON)") + +if(MI_GUARDED STREQUAL "FULL") + message(STATUS "Enable fine-grained sampling for guard pages (MI_GUARDED=FULL)") + list(APPEND mi_defines MI_GUARDED=2) +elseif(MI_GUARDED) + message(STATUS "Enable sampled guard pages (MI_GUARDED=${MI_GUARDED})") list(APPEND mi_defines MI_GUARDED=1) +else() + # message(STATUS "Disable guard pages (MI_GUARDED=${MI_GUARDED})") + list(APPEND mi_defines MI_GUARDED=0) endif() if(MI_NO_PADDING) @@ -473,6 +503,44 @@ else() message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, OFF, SHARED, or ON)") endif() +if(MI_STATS STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_STATS "FULL") + else() + set(MI_STATS "ON") + endif() +endif() + +if(MI_STATS STREQUAL "FULL") + message(STATUS "Enable detailed statistics including precise requested bytes (expensive) (MI_STATS=FULL)") + list(APPEND mi_defines MI_STATS=2) +elseif(MI_STATS) + message(STATUS "Enable detailed statistics (MI_STATS=${MI_STATS})") + list(APPEND mi_defines MI_STATS=1) +else() + message(STATUS "Disable detailed statistics (MI_STATS=${MI_STATS})") + list(APPEND mi_defines MI_STATS=0) +endif() + +if(MI_PROFILE STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_PROFILE "FULL") + else() + set(MI_PROFILE "ON") + endif() +endif() + +if(MI_PROFILE STREQUAL "FULL") + message(STATUS "Enable fine-grained sampled profiling (MI_PROFILE=FULL)") + list(APPEND mi_defines MI_PROFILE=2) +elseif(MI_PROFILE) + message(STATUS "Enable sampled profiling (MI_PROFILE=${MI_PROFILE})") + list(APPEND mi_defines MI_PROFILE=1) +else() + message(STATUS "Disable profiling (MI_PROFILE=${MI_PROFILE})") + list(APPEND mi_defines MI_PROFILE=0) +endif() + if(CMAKE_SYSTEM_NAME STREQUAL "WASI") message(STATUS "Enabling MI_FREE_USE_PAGEMAP since the target (WASI) does not support large OS aligned allocations well") set(MI_FREE_USE_PAGEMAP ON) @@ -483,13 +551,18 @@ if(MI_FREE_USE_PAGEMAP) endif() # deprecated -if(MI_OPT_FREE_SMALL) +if(MI_OPT_FREE_SMALL STREQUAL "DEFAULT") + # do nothing +elseif(MI_OPT_FREE_SMALL) if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") else() message(STATUS "Use aligned `mi_free_small` (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) endif() +else() + message(STATUS "Disable aligned `mi_free_small` (MI_OPT_FREE_SMALL=OFF)") + list(APPEND mi_defines MI_OPT_FREE_SMALL=0) endif() if(MI_XMALLOC) @@ -514,7 +587,7 @@ if(MI_DEBUG_TSAN) endif() if(MI_DEBUG_UBSAN) - if(CMAKE_BUILD_TYPE MATCHES "Debug") + if(CMAKE_BUILD_TYPE MATCHES "Debug" OR CMAKE_CONFIGURATION_TYPES) if(MI_USE_CXX) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Build with undefined-behavior sanitizer (MI_DEBUG_UBSAN=ON)") @@ -655,6 +728,13 @@ if(MI_ARCH MATCHES "riscv") endif() endif() +if(NOT WIN32) + CHECK_INCLUDE_FILES("execinfo.h" MI_HAS_EXECINFOH) + if(MI_HAS_EXECINFOH) + list(APPEND mi_defines MI_HAS_EXECINFOH=1) + endif() +endif() + # On Haiku use `-DCMAKE_INSTALL_PREFIX` instead, issue #788 # if(CMAKE_SYSTEM_NAME MATCHES "Haiku") # SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) @@ -818,7 +898,7 @@ endif() # set language for source files now if(MI_USE_CXX) set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) - set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c test/test-profile.c PROPERTIES LANGUAGE CXX ) endif() @@ -851,12 +931,26 @@ endif() if(MI_TRACK STREQUAL "ASAN") set(mi_libname "${mi_libname}-asan") endif() -string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $ ? -if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") - list(APPEND mi_defines MI_BUILD_RELEASE) +if (CMAKE_CONFIGURATION_TYPES) + # multi-config builds + # generator expression lower-case build type + set(mi_gx_build_type_lc $>) + # generator expression for "is this a release build?" + set(mi_gx_release $,$,$,$>) + + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${mi_gx_build_type_lc}") + list(APPEND mi_defines $<${mi_gx_release}:MI_BUILD_RELEASE>) + # append build type (e.g. -debug) if not a release version + set(mi_libname "${mi_libname}$<$:-${mi_gx_build_type_lc}>") else() - set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version + # single-config builds + string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $ ? + if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") + list(APPEND mi_defines MI_BUILD_RELEASE) + else() + set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version + endif() endif() string(REGEX REPLACE "^mimalloc[-]?" "" mi_libsuffix "${mi_libname}") @@ -877,7 +971,9 @@ endif() message(STATUS "") message(STATUS "Library name : ${mi_libname}") message(STATUS "Version : ${mi_version}.${mi_version_patch}") -message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +if(CMAKE_BUILD_TYPE_LC) + message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +endif() if(MI_USE_CXX) message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() @@ -986,11 +1082,7 @@ if (MI_BUILD_OBJECT) ) # Copy the generated object file (`static.o`) to the output directory (as `mimalloc.o`) - if(CMAKE_GENERATOR MATCHES "^Visual Studio.*$") - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/mimalloc-obj.dir/$/static${CMAKE_C_OUTPUT_EXTENSION}") - else() - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION}") - endif() + set(mimalloc-obj-static "$") set(mimalloc-obj-out "${CMAKE_CURRENT_BINARY_DIR}/${mi_libname}${CMAKE_C_OUTPUT_EXTENSION}") add_custom_command(OUTPUT ${mimalloc-obj-out} DEPENDS mimalloc-obj mimalloc-static COMMAND "${CMAKE_COMMAND}" -E copy "${mimalloc-obj-static}" "${mimalloc-obj-out}") add_custom_target(mimalloc-obj-target ALL DEPENDS ${mimalloc-obj-out}) @@ -1035,7 +1127,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - foreach(TEST_NAME api api-fill stress-heaps stress-subprocs stress) + foreach(TEST_NAME api api-fill stress-heaps stress-subprocs stress profile) add_executable(mimalloc-test-${TEST_NAME} test/test-${TEST_NAME}.c) target_compile_definitions(mimalloc-test-${TEST_NAME} PRIVATE ${mi_defines}) target_compile_options(mimalloc-test-${TEST_NAME} PRIVATE ${mi_cflags}) @@ -1047,10 +1139,12 @@ if (MI_BUILD_TESTS) else() message(STATUS "cannot build TSAN tests without MI_BUILD_SHARED being enabled") endif() - if(MI_GUARDED) - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) + if(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1024 $) elseif(TEST_NAME STREQUAL "stress-heaps") add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0 $) + elseif(NOT MI_PROFILE AND (TEST_NAME STREQUAL "profile")) + # don't add as a test else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) endif() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5094950e1..84ce387f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -127,7 +127,7 @@ jobs: CC: clang CXX: clang++ BuildType: debug-asan-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_TRACK_ASAN=ON Debug UBSAN Clang: CC: clang CXX: clang++ @@ -156,6 +156,7 @@ jobs: displayName: CTest env: MIMALLOC_GUARDED_SAMPLE_RATE: 1000 + MIMALLOC_VERBOSE: 2 # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) @@ -188,6 +189,7 @@ jobs: - script: ctest --verbose --timeout 240 workingDirectory: $(BuildType) displayName: CTest + # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-macos-$(BuildType) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 306ff6a68..349535cce 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 3) set(mi_version_minor 5) -set(mi_version_patch 1) +set(mi_version_patch 2) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/doc/release-notes.md b/doc/release-notes.md index 1e59f4a53..867484268 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,7 +1,7 @@ Binary and source releases for mimalloc. -- __v3__: recommended: latest mimalloc design that tends to use less memory then v2 but should perform similarly. -- __v2__: stable: most widely used. +- __v3__: recommended: latest mimalloc design that tends to use less memory then v2. +- __v2__: stable legacy. - __v1__: legacy. Notes: @@ -10,6 +10,6 @@ Notes: - Generally it is recommended to download sources (or use `vcpkg` etc.) and build mimalloc as part of your project. - Source releases can also be downloaded directly from github by the tag. - For example . + For example . - Binary releases include a release-, debug-, and secure build. - Linux binaries are built on Ubuntu 22. diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index cba6f88f0..9a17c0ae3 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -451,6 +451,8 @@ + + @@ -488,6 +490,7 @@ + diff --git a/ide/vs2022/mimalloc-lib.vcxproj.filters b/ide/vs2022/mimalloc-lib.vcxproj.filters index 0c23b2834..c140f17cf 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -70,6 +70,12 @@ Sources + + Sources + + + Sources + @@ -108,6 +114,9 @@ Headers + + Headers + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj index 25269e3d8..c2338bb2e 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -433,6 +433,7 @@ + @@ -480,6 +481,8 @@ + + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj.filters b/ide/vs2022/mimalloc-override-dll.vcxproj.filters index f57933c17..8bced90ae 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj.filters +++ b/ide/vs2022/mimalloc-override-dll.vcxproj.filters @@ -70,6 +70,12 @@ Sources + + Sources + + + Sources + @@ -108,6 +114,9 @@ Headers + + Headers + diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h new file mode 100644 index 000000000..5a1327ff7 --- /dev/null +++ b/include/mimalloc-profile.h @@ -0,0 +1,79 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2024-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PROFILE_H +#define MIMALLOC_PROFILE_H + +#include +#include // bool +#include // uint64_t + +//--------------------------------------------------------------------------- +// Initial support for profiling hooks; still experimental +//--------------------------------------------------------------------------- + +#define MI_PROFILE_SAMPLE_DATA_MAX_SIZE (1024) + +// User defined profiler; see `test_profile.c` for an example. +typedef struct mi_profiler_s mi_profiler_t; + +// Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) +typedef struct mi_profiler_sample_data_s { + size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`) + void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size` +} mi_profiler_sample_data_t; + +// Profiling callback invoked on each sampled allocation. +// If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`. +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap); + +// Profiling callback invoked on each sampled in-place re-allocation. +typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap); + +// Profiling callback invoked on a previously sampled allocation. +typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, const mi_heap_t* heap); + +// A profiler +// All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0. +struct mi_profiler_s { + void* reserved; // opaque; reserved by mimalloc + size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data) + size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`) + mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) + mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) + mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently) + // ... more user fields allowed +}; + +// Exported definitions +#ifdef __cplusplus +extern "C" { +#endif + +// attach a profiler to a particular heap only. +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler); + +// disable profiling for a particular heap; useful for a heap that the profiler uses itself for metadata. +mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap); + +// attach a profiler to any (current and future) heaps in a sub-process (unless those heaps disabled profiling) +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler); + +// attach a profiler to any heaps in the main sub-process +mi_decl_export bool mi_profile(mi_profiler_t* profiler); + +// start sampling +mi_decl_export bool mi_profiler_start(mi_profiler_t* profiler); + +// end sampling +mi_decl_export bool mi_profiler_stop(mi_profiler_t* profiler); + +#ifdef __cplusplus +} +#endif + +#endif // MIMALLOC_PROFILE_H diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index f8be41aa6..ab45a0715 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -49,7 +49,7 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNT(threads) /* number of threads */ \ MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ - MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \ + MI_STAT_COUNTER(malloc_requested) /* malloc requested bytes */ \ \ MI_STAT_COUNTER(mmap_calls) \ MI_STAT_COUNTER(commit_calls) \ @@ -75,11 +75,16 @@ typedef struct mi_stat_counter_s { /* only on v3 */ \ MI_STAT_COUNT(heaps) \ MI_STAT_COUNT(theaps) \ + MI_STAT_COUNT(pages_os_abandoned) /* pages in the os abandoned list (bad) */ \ + MI_STAT_COUNT(pages_os_allocated) /* pages allocated outside arenas (bad) */ \ MI_STAT_COUNTER(pages_reclaim_on_alloc) \ MI_STAT_COUNTER(pages_reclaim_on_free) \ MI_STAT_COUNTER(pages_reabandon_full) \ MI_STAT_COUNTER(pages_unabandon_busy_wait) \ - MI_STAT_COUNTER(heaps_delete_wait) + MI_STAT_COUNTER(heaps_delete_wait) \ + MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \ + MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \ + MI_STAT_COUNTER(profile_samples) /* total sampled profiled allocations */ \ // Size bins for chunks typedef enum mi_chunkbin_e { @@ -140,9 +145,6 @@ mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_ mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept; -// stats from a theap -mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; - // stats from a subprocess and its heaps aggregated mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept; mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL @@ -164,6 +166,12 @@ mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id mi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept; +// get theap stats (only thread safe on theaps belonging to the calling thread) +mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; + +// add the theap stats to the stats of the parent heap and clear the theap stats +mi_decl_export void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept; + #ifdef __cplusplus } #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index d38988613..77b2cfe2e 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30501 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30502 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -97,6 +97,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // size_t, wchar_t #include // bool +#include // inline assertions #ifdef __cplusplus extern "C" { @@ -120,7 +121,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* // Extended allocation functions // ------------------------------------------------------ #define MI_SMALL_WSIZE_MAX 128 -#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) +#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(size_t)) // mimalloc considers `sizeof(size_t)` as the machine word size mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -399,23 +400,36 @@ mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_the // Fast constant size allocations. // ------------------------------------------------------ +// Machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`) +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; + +// get the machine word size from a byte size. +static inline size_t mi_wsize_from_size(size_t size) { + return ((size + sizeof(size_t) - 1) / sizeof(size_t)); +} + static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_malloc_small(size); } else { return mi_malloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } } static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_zalloc_small(size); } else { return mi_zalloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } } -static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } } static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_zalloc(theap,size); } + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } } static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } } static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { - // assert(p!=NULL); + assert(p!=NULL); if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } } @@ -506,18 +520,19 @@ typedef enum mi_option_e { mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls - mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always + mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallow always, 0 allows if the page originated from the current theap, 1 allow always mi_option_page_full_retain, // retain N full (small) pages per size class (=2) mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4) mi_option_max_vabits, // max user space virtual address bits to consider (=48) mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0) mi_option_page_commit_on_demand, // commit page memory on-demand mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited)) - mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16) + mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=32) mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled. mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). - mi_option_arena_is_numa_local, // experimental + mi_option_arena_is_numa_local, // experimental: associate local numa node with an initial arena allocation + mi_option_collect_merges_stats, // on each theap collection, the theap stats are merged automatically with the parent heap _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index c7ce294f9..5aa0d8375 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -18,6 +18,14 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // LONG_MAX +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#define mi_init_struct_zero { } +#else +#define mi_decl_externc +#define mi_init_struct_zero { 0 } +#endif + // ------------------------------------------------------ // Size of a pointer. // We assume that `sizeof(void*)==sizeof(intptr_t)` diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fdd5c78be..80d9106a0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -40,6 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #if !MI_TRACK_ASAN #define mi_decl_forceinline __attribute__((always_inline)) inline @@ -55,10 +56,12 @@ terms of the MIT license. A copy of the license can be found in the file #else #define mi_decl_hidden __attribute__((visibility("hidden"))) #endif -#if (__GNUC__ >= 4) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) #define mi_decl_cold __attribute__((cold)) +#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz) #else #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #endif #elif __cplusplus >= 201103L // c++11 #define mi_decl_forceinline inline @@ -68,6 +71,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #else #define mi_decl_forceinline inline #define mi_decl_noinline @@ -76,6 +80,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #endif #if defined(__GNUC__) || defined(__clang__) @@ -89,7 +94,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_likely(x) (x) #endif -#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc +#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) #define mi_decl_maybe_unused __attribute__((unused)) #elif __cplusplus >= 201703L // c++17 #define mi_decl_maybe_unused [[maybe_unused]] @@ -101,15 +106,6 @@ terms of the MIT license. A copy of the license can be found in the file #define __has_builtin(x) 0 #endif -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#define mi_init_struct_zero { } -#else -#define mi_decl_externc -#define mi_init_struct_zero { 0 } -#endif - - #if defined(__EMSCRIPTEN__) && !defined(__wasi__) #define __wasi__ #endif @@ -152,10 +148,10 @@ void _mi_random_init(mi_random_ctx_t* ctx); void _mi_random_init_weak(mi_random_ctx_t* ctx); void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); -uintptr_t _mi_random_next(mi_random_ctx_t* ctx); -uintptr_t _mi_theap_random_next(mi_theap_t* theap); -uintptr_t _mi_os_random_weak(uintptr_t extra_seed); -static inline uintptr_t _mi_random_shuffle(uintptr_t x); +size_t _mi_random_next(mi_random_ctx_t* ctx); +size_t _mi_theap_random_next(mi_theap_t* theap); +size_t _mi_os_random_weak(size_t extra_seed); +static inline size_t _mi_random_shuffle(size_t x); // prim-tls.c void _mi_tls_slots_init(void); @@ -274,7 +270,8 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -282,9 +279,10 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free t void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... void _mi_deferred_free(mi_theap_t* theap, bool force); void _mi_page_free_collect(mi_page_t* page, bool force); -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); +void _mi_page_update_stats(mi_page_t* page); size_t _mi_page_stats_bin(const mi_page_t* page); // for stats size_t _mi_bin_size(size_t bin); // for stats @@ -325,12 +323,12 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); + // "free.c" void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); @@ -340,6 +338,20 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr bool _mi_page_is_valid(mi_page_t* page); #endif +// "sample-guarded.c" +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); +void _mi_page_unguard_all(mi_page_t* page); +void _mi_theap_guarded_init(mi_theap_t* theap); + +// "sample-profile.c" +mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +size_t _mi_theap_update_sample_rate(mi_theap_t* theap); + +mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); + // ------------------------------------------------------ // Assertions @@ -371,20 +383,32 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const ----------------------------------------------------------- */ // add to stat keeping track of the peak -void __mi_stat_increase(mi_stat_count_t* stat, size_t amount); -void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount); -void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount); -void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount); // adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total) -void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount); // counters can just be increased -void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); -void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); +static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); +} + +static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total += (int64_t)amount; +} + +static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total -= (int64_t)amount; +} #define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount) #define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount) @@ -399,15 +423,17 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); #define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) #define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) #define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) #define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) #define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } #define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } #define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } -#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } + /* ----------------------------------------------------------- pthread thread locals @@ -496,6 +522,12 @@ typedef struct mi_option_desc_s { // initialize a local variable to zero; use memset as compilers optimize constant sized memset's #define _mi_memzero_var(x) memset(&x,0,sizeof(x)) +// minimum +static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); } + +// maximum +static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); } + // Is `x` a power of two? (0 is considered a power of two) static inline bool _mi_is_power_of_two(uintptr_t x) { return ((x & (x - 1)) == 0); @@ -567,13 +599,6 @@ static inline bool mi_mem_is_zero(const void* p, size_t size) { return true; } -// Align a byte size to a size in _machine words_, -// i.e. byte size == `wsize*sizeof(void*)`. -static inline size_t _mi_wsize_from_size(size_t size) { - mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); - return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); -} - // Overflow detecting multiply #if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include // UINT_MAX, ULONG_MAX @@ -648,9 +673,9 @@ static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { return subproc; } -static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) { - mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - const size_t idx = _mi_wsize_from_size(size); +static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) { + mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize)); mi_assert_internal(idx < MI_PAGES_DIRECT); return theap->pages_free_direct[idx]; } @@ -661,7 +686,7 @@ static inline bool mi_theap_is_detached(mi_theap_t* theap) { static inline bool mi_theap_matches_thread(mi_theap_t* theap) { const mi_threadid_t tid = _mi_thread_id(); - return (theap==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); + return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); } /* ----------------------------------------------------------- @@ -905,11 +930,41 @@ static inline size_t mi_page_committed(const mi_page_t* page) { return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); } +static inline size_t mi_page_used(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return mi_xused_used_count(page->xused); +} + +static inline void mi_page_used_reset(mi_page_t* page) { + page->xused = mi_xused_used_reset(page->xused); +} + +static inline size_t mi_page_alloc_count(const mi_page_t* page) { + return mi_xused_alloc_count(page->xused); +} + +static inline size_t mi_page_last_used(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 32) & 0xFFFF; + #else + return page->xlast_used; + #endif +} + +static inline size_t mi_page_last_alloc(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 48) & 0xFFFF; + #else + return page->xlast_alloc; + #endif +} + + // are all blocks in a page freed? // note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. static inline bool mi_page_all_free(const mi_page_t* page) { mi_assert_internal(page != NULL); - return (page->used == 0); + return (mi_page_used(page)==0); } // are there immediately available blocks, i.e. blocks available on the free list. @@ -928,7 +983,7 @@ static inline bool mi_page_is_expandable(const mi_page_t* page) { static inline bool mi_page_is_full(const mi_page_t* page) { - const bool full = (page->reserved == page->used); + const bool full = (page->reserved == mi_page_used(page)); mi_assert_internal(!full || page->free == NULL); return full; } @@ -937,14 +992,14 @@ static inline bool mi_page_is_full(const mi_page_t* page) { static inline bool mi_page_is_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; - return (page->reserved - page->used <= frac); + return (page->reserved - mi_page_used(page) <= frac); } // is more than (n-1)/n'th of a page in use? static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) { if (page==NULL) return true; uint16_t frac = page->reserved / n; - return (page->reserved - page->used <= frac); + return (page->reserved - mi_page_used(page) <= frac); } @@ -1116,7 +1171,7 @@ static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { // are there any available blocks? static inline bool mi_page_has_any_available(const mi_page_t* page) { mi_assert_internal(page != NULL && page->reserved > 0); - return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); + return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL)); } // Owned? @@ -1132,14 +1187,26 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { /* ------------------------------------------------------------------- - Guarded objects + Guarded and profiled objects ------------------------------------------------------------------- */ -#if MI_GUARDED + +#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4) +#define MI_SAMPLE_COUNTDOWN_MAX (MI_MAX_ALLOC_SIZE) + +static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { + // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`. + mi_assert_internal(req_size <= SIZE_MAX/2); + // const size_t sample_countdown = theap->sample_countdown - req_size; + // return ((mi_ssize_t)sample_countdown < 0); + return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size); +} + // we always align guarded pointers in a block at an offset // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones #define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1)) #define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) -#endif + static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { #if MI_GUARDED @@ -1151,36 +1218,26 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* #endif } -#if MI_GUARDED -static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { - // this code is written to result in fast assembly as it is on the hot path for allocation - const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time.. - if mi_likely(count != 0) { - // no sample - theap->guarded_sample_count = count; - return false; - } - else { - // count == 0 - const size_t rate = theap->guarded_sample_rate; - if (rate == 0) { - return false; // don't write to an empty theap - } - else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { - // use guarded allocation - theap->guarded_sample_count = rate; // reset - return true; - } - else { - // failed size criteria, rewind count - theap->guarded_sample_count = 1; - return false; - } - } +static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) { +#if MI_GUARDED || MI_PROFILE + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + +static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; + return (mi_atomic_load_acquire(penabled) != 0); +} + +static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; + return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0); } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -#endif /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers @@ -1385,16 +1442,16 @@ static inline size_t _mi_memid_size(mi_memid_t memid) { // Fast "random" shuffle // ------------------------------------------------------------------- -static inline uintptr_t _mi_random_shuffle(uintptr_t x) { +static inline size_t _mi_random_shuffle(size_t x) { if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros -#if (MI_INTPTR_SIZE>=8) +#if (MI_SIZE_SIZE>=8) // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; x ^= x >> 27; x *= 0x94d049bb133111ebUL; x ^= x >> 31; -#elif (MI_INTPTR_SIZE==4) +#elif (MI_SIZE_SIZE==4) // by Chris Wellons, see: x ^= x >> 16; x *= 0x7feb352dUL; @@ -1408,108 +1465,82 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { // --------------------------------------------------------------------------------- // Provide our own `_mi_memcpy/set` for potential performance optimizations. -// -// For now, only on x64/x86 we optimize to `rep movsb/stosb`. -// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support -// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. -// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) - -extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c -extern mi_decl_hidden size_t _mi_cpu_stosb_max; - -static inline void mi_rep_movsb(void* dst, const void* src, size_t n) { - #if defined(__GNUC__) - __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory"); - #else - __movsb((unsigned char*)dst, (const unsigned char*)src, n); - #endif -} - -static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) { - #if defined(__GNUC__) - __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory"); - #else - __stosb((unsigned char*)dst, val, n); - #endif -} - -static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { - if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?) - mi_rep_movsb(dst, src, n); - return dst; - } - else { - return memcpy(dst, src, n); - } -} - -static inline void* _mi_memset(void* dst, int val, size_t n) { - if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127 - mi_rep_stosb(dst, (uint8_t)val, n); - return dst; - } - else { - return memset(dst, val, n); - } -} - -#else - static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { return memcpy(dst, src, n); } -static inline void* _mi_memset(void* dst, int val, size_t n) { +static inline void* _mi_memset(void* dst, int val, size_t n) { return memset(dst, val, n); } -#endif - -// ------------------------------------------------------------------------------- -// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned -// This is used for example in `mi_realloc`. -// ------------------------------------------------------------------------------- - -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) - -// On GCC/CLang we provide a hint that the pointers are word aligned. -static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); - return _mi_memcpy(adst, asrc, n); +static inline void* _mi_memset_backward(void* dst, int val, size_t n) { + memset((uint8_t*)dst - n, val, n); + return dst; } -static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - return _mi_memset(adst, val, n); +static inline void* _mi_memzero(void* dst, size_t n) { + return _mi_memset(dst, 0, n); } -#else +static inline void* _mi_memzero_backward(void* dst, size_t n) { + return _mi_memset_backward(dst, 0, n); +} -// Default fallback on `_mi_memcpy` static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - return _mi_memcpy(dst, src, n); + // on gcc/clang we can provide a hint that the pointers are word aligned. + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE); + return _mi_memcpy(adst, asrc, n); } static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - return _mi_memset(dst, val, n); -} - -#endif - -static inline void* _mi_memzero(void* dst, size_t n) { - return _mi_memset(dst, 0, n); + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + return _mi_memset(adst, val, n); } static inline void* _mi_memzero_aligned(void* dst, size_t n) { return _mi_memset_aligned(dst, 0, n); } +// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes. +static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + mi_assert_internal(bsize > 0); + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); + mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + + // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) + // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler + // (compile with -DMI_USE_MEMZERO16X=0 to disable this) + #if !defined(MI_USE_MEMZERO16X) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) + if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) + *((size_t*)dst) = 0; + return dst; + } + mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); + uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) + if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32) + const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8) + _mi_memzero(start, 2*MI_SIZE_SIZE); + _mi_memzero(start+ofs, 2*MI_SIZE_SIZE); + _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE); + _mi_memzero_backward(end, 2*MI_SIZE_SIZE); + return dst; + } + if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64) + _mi_memzero(start, 8*MI_SIZE_SIZE); + _mi_memzero_backward(end, 8*MI_SIZE_SIZE); + return dst; + } + #endif + // fallback to regular memset for larger sizes + void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE); + return _mi_memzero_aligned(wdst, bsize); +} + #endif // MI_INTERNAL_H diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 13f5860f9..da78d0193 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -35,7 +35,7 @@ static inline mi_theap_t* _mi_theap_cached(void); // static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) +static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) // Default TLS model @@ -118,7 +118,7 @@ static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS - #elif (MI_INTPTR_SIZE==4) + #elif (MI_SIZE_SIZE==4) __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI #else __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS @@ -301,9 +301,9 @@ static inline mi_theap_t* _mi_theap_cached(void) { // We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available) // See for the offsets. #if MI_SIZE_SIZE==4 -#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_INTPTR_SIZE) +#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE) #else -#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_INTPTR_SIZE) +#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE) #endif extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot; @@ -396,6 +396,16 @@ static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { return _mi_heap_theap_get_or_init(heap); } +static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + return NULL; +} + // Get the theap belonging to a heap without creating it if it is not yet initialized. static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { mi_theap_t* theap = _mi_theap_cached(); @@ -411,10 +421,10 @@ static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { // Find the associated theap or NULL if it does not exist (during shutdown) // Should be fast as it is called in `free.c:mi_free_try_collect`. -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { +static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page) { mi_heap_t* const heap = mi_page_heap(page); mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); - if (theap==NULL) return NULL; + if (theap==NULL || theap->tld==NULL /* heap destroy */) return NULL; if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap)); // note: for pages allocated by a detached theap, the returned theap may not be detached diff --git a/include/mimalloc/track.h b/include/mimalloc/track.h index 753363366..e6a57c8ec 100644 --- a/include/mimalloc/track.h +++ b/include/mimalloc/track.h @@ -133,7 +133,7 @@ defined, undefined, or not accessible at all: #endif -#if MI_PADDING +#if MI_PADDING_CHECK_BYTES && !MI_GUARDED #define mi_track_malloc(p,reqsize,zero) \ do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)==(reqsize)); \ diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 308c17bd3..21e787b43 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -24,12 +24,13 @@ terms of the MIT license. A copy of the license can be found in the file #include +#include #include // ptrdiff_t #include // uintptr_t, uint16_t, etc #include // bool #include // SIZE_MAX etc. #include // error codes -#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations +#include "bits.h" // size defines (MI_SIZE_SIZE etc), bit operations #include "atomic.h" // _Atomic primitives // Minimal alignment necessary. On most platforms 16 bytes are needed @@ -38,6 +39,9 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif +#if MI_MAX_ALIGN_SIZE < MI_INTPTR_SIZE +#error MI_MAX_ALIGN_SIZE must be at least MI_INTPTR_SIZE +#endif // ------------------------------------------------------ // Variants @@ -51,8 +55,8 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_TRACK_ASAN 1 // #define MI_TRACK_ETW 1 -// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). -// #define MI_STAT 1 +// Define MI_STATS as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). +// #define MI_STATS 1 // Define MI_SECURE to enable security mitigations // #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption @@ -65,8 +69,6 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_SECURE 0 #endif -#define MI_PADDING 0 - // Define MI_DEBUG for assertion and invariant checking // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON) // #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON) @@ -79,24 +81,44 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking) -#ifndef MI_STAT +// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=slightly more expensive tracking of precise requested bytes) +#ifndef MI_STATS #if (MI_DEBUG>0) -#define MI_STAT 2 +#define MI_STATS 2 #else -#define MI_STAT 0 +#define MI_STATS 1 #endif #endif +// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates(<64 KiB), a tad more expensive) +#ifndef MI_PROFILE +#define MI_PROFILE 1 +#endif + // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) #if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL #define MI_GUARDED 1 #endif +// For profiling or guarded pages we need to sample every once in a while. (0=no sampling, 1=fast, 2=allow fine grained sample rates(<64 KiB), a tad more expensive) +#ifndef MI_SAMPLE +#if MI_PROFILE>1 || MI_GUARDED>1 + #define MI_SAMPLE 2 +#elif MI_PROFILE || MI_GUARDED + #define MI_SAMPLE 1 +#else + #define MI_SAMPLE 0 +#endif +#endif + // Reserve extra padding at the end of each block to be more resilient against theap block overflows. // The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +#if !defined(MI_PADDING) +#if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 +#else +#define MI_PADDING 0 +#endif #endif // Check for byte-precise buffer overflow? @@ -147,7 +169,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP #if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_IS_ALIGNED 1 -#define MI_PAGE_META_ALIGNED_CHUNKS MI_INTPTR_SIZE +#define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" #endif @@ -159,7 +181,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) #if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 #define MI_PAGE_META_SMALL_IS_ALIGNED 0 -#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // cannot be guarded as that may allocate large blocks for small allocations +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail) #define MI_PAGE_META_SMALL_IS_ALIGNED 1 #else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 @@ -392,8 +414,28 @@ typedef size_t mi_page_flags_t; // This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; -// Convenience -typedef size_t mi_used_t; +// We store the currently used block count together with the total malloc call count as 16-bit numbers. +// This is done for better codegen `mi_malloc/mi_free` (where we can increment both at once as `used_alloc += 0x10001` for example). +// The `last_used` is the `used` count since the statistics are last updated; on 32-bit platforms this +// is a separate field in `mi_page_t` but on 64-bit we use the upper 32-bits to store it. +// We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even +// in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`). +typedef union mi_used_s { + size_t used_alloc; // used + alloc_count + // the following struct is unused but nice for debugging + struct { + uint16_t used_count; + uint16_t alloc_count; + #if MI_SIZE_SIZE >= 8 + uint16_t last_used; + uint16_t last_alloc; + #endif + } debug_le; +} mi_used_t; + +static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } +static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } +static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: @@ -428,7 +470,11 @@ typedef struct mi_page_s { #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - mi_used_t used; // number of blocks in use (including blocks in `thread_free`) + mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics) + #if MI_SIZE_SIZE < 8 + uint16_t xlast_used; // for statistics; on 64-bit platforms it is in bits 32..47 of xused. + uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused. + #endif mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) size_t block_size; // const: size available in each block (always `>0`) @@ -450,12 +496,9 @@ typedef struct mi_page_s { #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary - // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 - // uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align #endif } mi_page_t; - // ------------------------------------------------------ // Object sizes // ------------------------------------------------------ @@ -548,7 +591,7 @@ typedef struct mi_padding_s { uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes) } mi_padding_t; #define MI_PADDING_SIZE (sizeof(mi_padding_t)) -#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) +#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_SIZE_SIZE - 1) / MI_SIZE_SIZE) #else #define MI_PADDING_SIZE 0 #define MI_PADDING_WSIZE 0 @@ -562,11 +605,30 @@ struct mi_theap_s { // put in front for fast small allocations mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + // less frequently accessed fields mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) - _Atomic(size_t) refcount; // reference count + _Atomic(size_t) refcount; // reference count (needed for safe heap destroy) + // config + long page_full_retain; // how many full pages can be retained per queue (before abandoning them) + bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages + bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint + bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + + // sampling + size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`) + size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode) + uint64_t sample_requested; // total allocated/requested bytes since the last sample + size_t profile_sample_rate; // sampling rate in requested bytes for profiling + size_t profile_sample_countdown; // countdown in requested bytes for profiling + size_t guarded_sample_rate; // sampling rate in requested bytes for guarded objects + size_t guarded_sample_countdown; // countdown in requested bytes for guarded objects + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + + // stats unsigned long long heartbeat; // monotonic heartbeat count mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. @@ -576,21 +638,13 @@ struct mi_theap_s { long generic_count; // how often is `_mi_malloc_generic` called? long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting? + // theaps belong to heaps and threads mi_theap_t* tnext; // list of theaps in this thread mi_theap_t* tprev; mi_theap_t* hnext; // list of theaps of the owning `heap` mi_theap_t* hprev; - long page_full_retain; // how many full pages can be retained per queue (before abandoning them) - bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages - bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint - bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` - #if MI_GUARDED - size_t guarded_size_min; // minimal size for guarded objects - size_t guarded_size_max; // maximal size for guarded objects - size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) - size_t guarded_sample_count; // current sample count (counting down to 0) - #endif + // page queues mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics @@ -627,6 +681,8 @@ typedef struct mi_heap_s { mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields) mi_lock_t theaps_lock; // lock for the theaps list operations + _Atomic(mi_profiler_t*) profiler; + _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations) @@ -659,7 +715,7 @@ struct mi_subproc_s { mi_decl_align(8) // needed on some 32-bit platforms _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged - _Atomic(mi_heap_t*) heap_main; // main heap for this sub process + _Atomic(mi_heap_t*) heap_main; // main heap for this sub process mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; @@ -671,6 +727,8 @@ struct mi_subproc_s { _Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|) _Atomic(size_t) heap_total_count; // total created heaps in this sub-process + _Atomic(mi_profiler_t*) profiler; + mi_memid_t memid; // provenance of this memory block (meta or static) mi_subproc_t* parent; // subproc in which this one was allocated mi_decl_align(8) // needed on some 32-bit platforms diff --git a/readme.md b/readme.md index dcfc524b1..8320d5b20 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.5.1` (2026-09-01) recommended. -Latest v2 release: `v2.5.1` (2026-09-01) stable, legacy. -Latest v1 release: `v1.15.1` (2026-09-01) legacy. +Latest release : `v3.5.2` (2026-09-12) recommended. +Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy. +Latest v1 release: `v1.15.2` (2026-09-12) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -90,13 +90,16 @@ New development is mostly on v3, while v1 and v2 are maintained with security an ### Releases +* 2026-09-12, `v3.5.2`, `v2.5.2`, `v1.15.2`: (v3) Reduced cache contention, improved `mi_malloc_csize`, + improved zero'ing of small blocks, `mi_wmalloc` variants for runtime systems, always enable detailed + statistics. Experimental support for profiling hooks (`mimalloc-profile.h`). * 2026-09-01: Added a readme section on [getting the best performance](#getting-the-best-performance). * 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. - (v3) Added `mi_free_small(_nonnull)` for runtimes, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. + (v3) Added `mi_free_small(_nonnull)` for fast small allocations, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. (v3) Default to `-DMI_ALLOW_THP=FULL` on Linux, where we never split transparent huge pages (THP) during purging (#1282). This may increase rss but can also improve performance. Use `-DMI_ALLOW_THP=OFF` to disable the use of THP. - (v3) Improved cache behavior for small allocations. Improved riscV suppor, add riscV to CI. Various small build fixes. + (v3) Improved cache behavior for small allocations. Improved riscV support. Various small build fixes. * 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for faster load-acquire, increase retired page count from 1 to 3, faster double free @@ -125,27 +128,6 @@ New development is mostly on v3, while v1 and v2 are maintained with security an * 2026-04-20, `v1.9.9`, `v2.3.1`, `v3.3.1`: various bug and security fixes. Special thanks to @jinpzhanAMD, @res2k, and @GoldJohnKing for their help in improving Windows finalization, and @Zoxc for his help in finding various issues. -* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, - fix visiting of full pages during collection (performance), - fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode, - always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the - static CRT (by @Noxybot), fix TLS slot leak on Windows (v3), enable clean DLL load/unload with statically linked - mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated - objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: - fix offset for `mi_heap_realloc_aligned`, fix `mi_(w)dupenv_s` buffer size, fix potential overflow in size options, - and error codes for `mi_reallocarr(ay)`. -* 2026-02-03, `v3.2.8` (rc3): Fix thread reinitialize issue on macOS. Fix SIMD codegen bug on older - GCC versions. Extend Windows TLS slot limit from 64 to 1088. Report commit statistics more precise. - Fixes issue in free-page search in arenas. -* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated. - For v3 various bug and performance fixes. Fix Debian 32-bit compilation. -* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including - true first-class heaps where one can allocate in heap from any thread, and track statistics per heap as well. - Added `MIMALLOC_ALLOW_THP` option. This is by default enabled except on Android. When THP is detected on v3, - mimalloc will set the `MIMALLOC_MINIMAL_PURGE_SIZE` to 2MiB to avoid breaking up potential THP huge pages. - v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. - Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning - allocated sizes on allocation and free. * [Older release notes](#older-release-notes) @@ -985,6 +967,27 @@ provided by the bot. You will only need to do this once across all repos using o # Older Release Notes +* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, + fix visiting of full pages during collection (performance), + fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode, + always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the + static CRT (by @Noxybot), fix TLS slot leak on Windows (v3), enable clean DLL load/unload with statically linked + mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated + objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: + fix offset for `mi_heap_realloc_aligned`, fix `mi_(w)dupenv_s` buffer size, fix potential overflow in size options, + and error codes for `mi_reallocarr(ay)`. +* 2026-02-03, `v3.2.8` (rc3): Fix thread reinitialize issue on macOS. Fix SIMD codegen bug on older + GCC versions. Extend Windows TLS slot limit from 64 to 1088. Report commit statistics more precise. + Fixes issue in free-page search in arenas. +* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated. + For v3 various bug and performance fixes. Fix Debian 32-bit compilation. +* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including + true first-class heaps where one can allocate in heap from any thread, and track statistics per heap as well. + Added `MIMALLOC_ALLOW_THP` option. This is by default enabled except on Android. When THP is detected on v3, + mimalloc will set the `MIMALLOC_MINIMAL_PURGE_SIZE` to 2MiB to avoid breaking up potential THP huge pages. + v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. + Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning + allocated sizes on allocation and free. * 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 111d63a8b..3a4383e0b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -27,7 +27,7 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { return ok; } -#if MI_GUARDED +#if 0 static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept { // use over allocation for guarded blocksl #if MI_THEAP_INITASNULL @@ -47,21 +47,22 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m mi_assert_internal(_mi_is_aligned(p, alignment)); return p; } - +#endif +#if MI_GUARDED static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - #if MI_THEAP_INITASNULL - if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } - #endif - const size_t rate = theap->guarded_sample_rate; + // #if MI_THEAP_INITASNULL + // if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } + // #endif + // const size_t rate = theap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` - if (rate != 0) { theap->guarded_sample_rate = 0; } - void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); - if (rate != 0) { theap->guarded_sample_rate = rate; } + // if (rate != 0) { theap->guarded_sample_rate = 0; } + void* p = _mi_theap_malloc_zero(theap, size, zero, 0, ppage); + // if (rate != 0) { theap->guarded_sample_rate = rate; } return p; } #else static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - return _mi_theap_malloc_zero(theap, size, zero, ppage); + return _mi_theap_malloc_zero(theap, size, zero, 0, ppage); } #endif @@ -85,7 +86,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block + p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { @@ -112,19 +113,18 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // (we can access the page though since the just allocated pointer keeps it alive) if (aligned_p != p) { mi_page_set_has_interior_pointers(page, true); - // if (usable!=NULL) { - // mi_assert_internal(*usable > adjust); - // if (*usable > adjust) { *usable = *usable - adjust; } - // mi_assert_internal(*usable >= size); - // } - #if MI_GUARDED + + #if MI_GUARDED || MI_PROFILE // set tag to aligned so mi_usable_size works with guard pages + // note: we could have gotten a profiled or guarded block but that is still ok + // and in such case the MI_BLOCK_TAG_ALIGNED is just not at the start of the real heap block if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; block->next = MI_BLOCK_TAG_ALIGNED; } #endif - _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); + mi_block_t* const base = _mi_page_ptr_unalign(page, p); + _mi_padding_shrink(page, base, adjust + size); } // todo: expand padding if overallocated ? @@ -134,22 +134,13 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); #if MI_DEBUG > 1 mi_page_t* const apage = _mi_ptr_page(aligned_p); - void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); - mi_assert_internal(p == unalign_p); + mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_sampled(unalign_p, aligned_p)); #endif - // now zero the block if needed - //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { - // // for the tracker, on huge aligned allocations only from the start of the large block is defined - // mi_track_mem_undefined(aligned_p, size); - // if (zero) { - // _mi_memzero_(aligned_p, mi_usable_size(aligned_p)); - // } - //} - if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); - #if MI_GUARDED + #if MI_GUARDED || MI_PROFILE mi_track_mem_defined(p, sizeof(mi_block_t)); #endif } @@ -169,17 +160,30 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // use regular allocation if it is guaranteed to fit the alignment constraints. // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. - if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, ppage); - mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); - const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; - if mi_likely(is_aligned_or_null) { - return p; - } - else { - // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. - mi_assert(false); - mi_free(p); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + { + #if MI_SAMPLE // only try if we would not take a sample + if mi_likely(!mi_theap_should_sample(theap,size)) + #endif + { + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) + { + mi_page_t* page = NULL; + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); + if (ppage!=NULL) { *ppage = page; } + const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + if mi_likely(is_aligned_or_null) { + return p; + } + else { + // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. + // (but it can happen if this allocation just so happens to be sampled but we checked for that) + mi_assert_internal(false); // p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p)); + mi_free(p); + } + } } } @@ -201,36 +205,33 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con return mi_error_bad_alignment(size, alignment, offset); } - #if MI_GUARDED - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) - #endif - if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) { - return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero, ppage); - } - #endif - // try first if there happens to be a small block available with just the right alignment // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already // naturally aligned this can be often the case. - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) - #endif + if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { - if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { - const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); - if mi_likely(page->free != NULL) { - const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; - if mi_likely(is_aligned) - { - if (ppage!=NULL) { *ppage = page; } - void* p = _mi_page_malloc_zero(theap, page, padsize, zero); - mi_assert_internal(p != NULL); - mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); - mi_track_malloc(p, size, zero); - return p; + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + { + #if MI_SAMPLE // check if we shouldn't take a sample + if mi_likely(!mi_theap_should_sample(theap,size)) + #endif + { + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; + mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false); + if mi_likely(page->free != NULL) { + const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; + if mi_likely(is_aligned) + { + if (ppage!=NULL) { *ppage = page; } + void* p = _mi_page_malloc_zero(theap, page, padsize, zero); + mi_assert_internal(p != NULL); + mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + mi_track_malloc(p, size, zero); + return p; + } } } } diff --git a/src/alloc-override.c b/src/alloc-override.c index 04da5f293..bcb96b52d 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -293,7 +293,7 @@ typedef void* mi_nothrow_t; void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&) void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&) - #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 + #if (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } @@ -302,7 +302,7 @@ typedef void* mi_nothrow_t; void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - #elif (MI_INTPTR_SIZE==4) + #elif (MI_SIZE_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } diff --git a/src/alloc.c b/src/alloc.c index 945ea0ca3..ded0a00c5 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -30,10 +30,8 @@ terms of the MIT license. A copy of the license can be found in the file static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept { const size_t bsize = mi_page_usable_block_size(page); mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>=2) + const ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); - #endif mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess padding->canary = mi_ptr_encode_canary(page,block,page->keys); padding->delta = (uint32_t)(delta); @@ -45,30 +43,26 @@ static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_blo } #endif } -#else -static mi_decl_maybe_unused mi_decl_noinline void* mi_block_zero(mi_block_t* block, size_t bsize) mi_attr_noexcept { - // const size_t bsize = mi_page_usable_block_size(page); - _mi_memzero_aligned(block, bsize); - return block; -} #endif // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept { if (page->block_size != 0) { // not the empty theap mi_assert_internal(mi_page_block_size(page) >= size); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); } // check the free list mi_block_t* const block = page->free; - const mi_used_t used = page->used; + mi_used_t xused = page->xused; + xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count #if defined(__GNUC__) - __asm("" : : : "memory"); // always load the `used` field before the test + __asm("" : : : "memory" ); // always load the `used` field before the test #endif if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); @@ -79,14 +73,29 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // pop from the free list mi_block_t* next = mi_block_next(page,block); mi_track_mem_undefined(block,sizeof(*block)); + #if MI_SECURE if (!zero) block->next = 0; // don't leak internal data #endif + page->free = next; - page->used = used+1; + page->xused = xused; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + #if MI_SAMPLE==2 + const size_t req_size = size - MI_PADDING_SIZE; + mi_assert_internal(theap->sample_countdown == sample_countdown); // we pass it to improve codegen + mi_assert_internal(theap->sample_countdown >= req_size); + theap->sample_countdown = sample_countdown - req_size; + #else + MI_UNUSED(sample_countdown); + #endif + + #if MI_STATS>=2 + mi_theap_stat_counter_increase(theap,malloc_requested,size - MI_PADDING_SIZE); + #endif + #if MI_DEBUG>3 if (page->free_is_zero && size > sizeof(*block)) { mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); @@ -100,18 +109,8 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_track_mem_undefined(block, bsize); // track per-block statistics - #if (MI_STAT>0) - if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { - mi_theap_stat_increase(theap, malloc_normal, bsize); - #if (MI_STAT>1) - mi_theap_stat_counter_increase(theap, malloc_normal_count, 1); - const size_t bin = _mi_bin(bsize); - mi_theap_stat_increase(theap, malloc_bins[bin], 1); - mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE); - #endif - } - #endif - + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + // in debug mode initialize with 0xD0 #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } @@ -120,7 +119,11 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // zero the block? note: we need to zero the full block size (issue #63) if mi_unlikely(zero) { if (!page->free_is_zero) { - _mi_memzero_aligned(block,bsize); + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + #if !MI_PADDING + return // use tail-call + #endif + _mi_memzero_block(block,bsize); } else { block->next = 0; @@ -138,31 +141,32 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { - return mi_page_malloc_zero(theap, page, size, zero, NULL); + return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL); } -// main allocation primitives for small and generic allocation - // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { + size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local #endif - #if (MI_PADDING || MI_GUARDED) - if mi_unlikely(size == 0) { size = sizeof(void*); } + #if (MI_PADDING) // || MI_GUARDED) + if mi_unlikely(xsize == 0) { xsize = (is_wsize ? 1 : MI_SIZE_SIZE); size += MI_SIZE_SIZE; } #endif - #if MI_GUARDED - if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) { - return _mi_theap_malloc_guarded(theap, size, zero, ppage); - } + + // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) + #if MI_SAMPLE==2 + if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sampled(theap,size,zero,ppage); } #endif + + // get page in constant time + mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize); - // get page in constant time, and allocate from it - mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); - void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); + // and allocate + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -176,14 +180,6 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non // internal generic allocation static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { - #if MI_GUARDED - #if MI_THEAP_INITASNULL - if (theap!=NULL) - #endif - if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) { - return _mi_theap_malloc_guarded(theap, size, zero, ppage); - } - #endif #if !MI_THEAP_INITASNULL mi_assert(theap!=NULL); #endif @@ -201,46 +197,64 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL if (theap!=NULL) { - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall + return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); #endif } - // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(theap, size, false, NULL); + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small(_mi_theap_default(), size); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); +} + +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL); + // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } + // but the following prevents using a stack frame. We use this to optimize some select functions only. + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); } + else { return mi_heap_init_malloc_small(heap, size); } } // The main internal allocation functions static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { + mi_assert(theap!=NULL); // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +extern mi_decl_forceinline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) @@ -249,57 +263,83 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); -} // Main allocation functions mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, false, 0, NULL); + mi_assert(theap!=NULL); + return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_theap_malloc(_mi_theap_default(), size); + return _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, NULL); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); + // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } + else { return mi_heap_init_malloc(heap, size); } + } // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(theap, size, true, NULL); + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } -mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, true, NULL); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); +} +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } + else { return mi_heap_init_zalloc_small(heap, size); } +} mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(theap, size, true, NULL); + return _mi_theap_malloc_zero(theap, size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); + return _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, NULL); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, true, 0, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); + // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, true, 0, NULL); } + else { return mi_heap_init_zalloc(heap, size); } } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { @@ -329,25 +369,25 @@ static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { // Return usable size mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, true, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page); return mi_ublock_size(p,page,pblock_size); } @@ -430,7 +470,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi } // note: we don't zero allocate upfront so we only zero initialize the expanded part mi_page_t* newpage; // use block_size for zero-ing, issue #763 - void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,&newpage); + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); if mi_likely(newp != NULL) { if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } const size_t copy_size = (newsize > size ? size : newsize); @@ -874,95 +914,6 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } } -#if MI_GUARDED -// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). -// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) -// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` - -static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { - // todo: we can still make padding work by moving it out of the guard page area - mi_page_t* const page = _mi_ptr_page(block); - mi_page_set_has_interior_pointers(page, true); - block->next = MI_BLOCK_TAG_GUARDED; - - // set guard page at the end of the block - const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` - const size_t os_page_size = _mi_os_page_size(); - mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); - if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { - // should never happen - mi_free(block); - return NULL; - } - uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; - // note: the alignment of the guard page relies on blocks being os_page_size aligned which - // is ensured in `mi_arena_page_alloc_fresh`. - mi_assert_internal(_mi_is_aligned(block, os_page_size)); - mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); - if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { - const bool ok = _mi_os_protect(guard_page, os_page_size); - if mi_unlikely(!ok) { - _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); - } - } - else { - _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); - } - - // align pointer just in front of the guard page - size_t offset = block_size - os_page_size - obj_size; - mi_assert_internal(offset > sizeof(mi_block_t)); - if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { - // give up to place it right in front of the guard page if the offset is too large for unalignment - offset = MI_PAGE_MAX_OVERALLOC_ALIGN; - } - uint8_t* const p = (uint8_t*)block + offset; - mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); - if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } - mi_track_align(block, p, offset, obj_size); - mi_track_mem_defined(block, sizeof(mi_block_t)); - return p; -} - -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept -{ - // allocate multiple of page size ending in a guard page - // ensure minimal alignment requirement? - if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow - _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); - return NULL; - } - const size_t os_page_size = _mi_os_page_size(); - const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); - const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); - const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); - if (block==NULL) return NULL; - size_t usable_size = 0; - void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); - if (p == NULL) return NULL; - if (zero) { - _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) - } - - // stats - mi_track_malloc(p, usable_size, zero); - if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } - mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); - #if MI_STAT>1 - // adjust stats to only count the allocated size of the block (and not the guard page) - mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size); - mi_theap_stat_increase(theap, malloc_requested, size); - #endif - #if MI_DEBUG>3 - if (zero) { - mi_assert_expensive(mi_mem_is_zero(p, size)); - } - #endif - return p; -} -#endif - // ------------------------------------------------------ // ensure explicit external inline definitions are emitted! // ------------------------------------------------------ @@ -971,7 +922,6 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, void* _mi_externs[] = { (void*)&_mi_page_malloc_zero, (void*)&_mi_theap_malloc_zero, - (void*)&_mi_theap_malloc_zero_ex, (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, diff --git a/src/arena.c b/src/arena.c index 2bda823bc..ffded1067 100644 --- a/src/arena.c +++ b/src/arena.c @@ -864,6 +864,7 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); } #endif + if (start!=NULL) { mi_heap_stat_increase(heap,pages_os_allocated,1); } } if (start == NULL) return NULL; @@ -978,9 +979,9 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou if (page_meta!=NULL) { mi_assert_internal(page_meta->block_size == 0); #if MI_PAGE_META_SMALL_IS_ALIGNED - // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, + // if `block_size <= MI_SMALL_MAX_OBJ_SIZE` we put the page info in front of the slice, // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (block_size <= MI_SMALL_SIZE_MAX) { + if (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) { // put page info in front of the slice page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); @@ -994,7 +995,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ #endif - if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && + if (block_size >= MI_SIZE_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment @@ -1288,6 +1289,9 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); } } + else { + mi_heap_stat_decrease(page->heap, pages_os_allocated, 1); + } if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid); } @@ -1304,6 +1308,8 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { mi_heap_t* const heap = mi_page_heap(page); mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); mi_theapx_stat_decrease(heap, current_theapx, pages, 1); + _mi_page_free_collect(page,false); // update used count for cross-thread free's + _mi_page_update_stats(page); // and update the stats mi_arenas_page_free_prim(page); } @@ -1321,6 +1327,11 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_theap_matches_thread(current_theapx)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); + // note: somewhat expensive to update here, but might be good as then we attribute + // the current allocations/frees to the current thread/theap. Otherwise it might be + // reclaimed later in another thread/theap and those allocations/frees get attributed there... + _mi_page_update_stats(page); + // add to abandoned? mi_heap_t* heap = mi_page_heap(page); if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { @@ -1359,6 +1370,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { if (page->next != NULL) { page->next->prev = page; } heap->os_abandoned_pages = page; } + mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1); } mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); mi_abandoned_page_unown(page, current_theapx); @@ -1428,6 +1440,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { page->next = NULL; page->prev = NULL; } + mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1); } } mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); @@ -1904,7 +1917,7 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com } mi_memid_t memid; void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); - if (start == NULL) return ENOMEM; + if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { _mi_os_free_ex(subproc, start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); @@ -1983,7 +1996,7 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { static int mi_page_commit_usage(mi_page_t* page) { const size_t committed_size = mi_page_committed(page); - const size_t used_size = page->used * mi_page_block_size(page); + const size_t used_size = mi_page_used(page) * mi_page_block_size(page); return (int)(used_size * 100 / committed_size); } @@ -2554,7 +2567,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_owned(page)); - if (page->used==0) { + if (mi_page_used(page)==0) { // free the page _mi_arenas_page_free(page, theap); } @@ -2563,7 +2576,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are _mi_page_unguard_all(page); // remove potential interior guard pages #endif // destroy the page - page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case + mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case _mi_arenas_page_free(page, theap); } else { diff --git a/src/bitmap.h b/src/bitmap.h index ba8224b8a..79eda77fa 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -71,7 +71,7 @@ typedef size_t mi_bfield_t; #define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit -// some compiler (msvc in C mode) cannot have expressions in the alignment attribute +// some compilers (msvc in C mode) cannot have expressions in the alignment attribute #if MI_BCHUNK_SIZE==64 #define mi_decl_bchunk_align mi_decl_align(64) #elif MI_BCHUNK_SIZE==32 diff --git a/src/free.c b/src/free.c index 5d033aab4..cf352849c 100644 --- a/src/free.c +++ b/src/free.c @@ -16,8 +16,6 @@ terms of the MIT license. A copy of the license can be found in the file mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); -static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); - // ------------------------------------------------------ // Free @@ -25,28 +23,29 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // regular free of a (thread local) block pointer // fast path written carefully to prevent spilling on the stack -static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full) +static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full) { // checks size_t usable_size; if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding - if (track_stats) { - mi_stat_free(page, block); - mi_track_free_size(block, usable_size); - } + // mi_stat_free(page, block); + mi_track_free_size(block, usable_size); + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); #endif // actual free: push on the local free list - const mi_used_t used = page->used - 1; + mi_used_t xused = page->xused; + xused.used_alloc--; // decrement used count mi_block_set_next(page, block, page->local_free); - page->used = used; + page->xused = xused; page->local_free = block; - if mi_unlikely(used==0) { + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + if mi_unlikely(mi_xused_used_count(xused) == 0) { // is used count zero ? if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) _mi_page_retire(page); } @@ -57,16 +56,16 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool } // Forward declaration for multi-threaded collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept; +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept; // Free a block multi-threaded -static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept +static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept { size_t usable_size; if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt // adjust stats (after padding check ) - mi_stat_free(page, block); // stat_free may access the padding + // mi_stat_free(page, block); // stat_free may access the padding mi_track_free_size(block, usable_size); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); @@ -81,19 +80,16 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { - mi_block_set_next(page, block, mi_tf_block(tf_old)); - const bool new_owned = (allow_collect ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned - tf_new = mi_tf_create(block, new_owned); + mi_block_set_next(page, block, mi_tf_block(tf_old)); + tf_new = mi_tf_create(block, true /* try to own it */ ); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough? // and atomically try to collect the page if it was abandoned - if (allow_collect) { - const bool is_owned_now = !mi_tf_is_owned(tf_old); - if (is_owned_now) { - mi_assert_internal(mi_page_is_abandoned(page)); - mi_free_try_collect_mt(page,block); - } - } + const bool is_owned_now = !mi_tf_is_owned(tf_old); + if (is_owned_now) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_free_try_collect_mt(page,block,allow_reclaim); + } } @@ -101,7 +97,7 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was // note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the // `page_woffset` and `block_size` fields; however these are constant and the page won't be // deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently. -mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { +static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const void* p, size_t* poffset) { mi_assert_internal(page!=NULL && p!=NULL); const size_t diff = (uint8_t*)p - mi_page_start(page); @@ -110,9 +106,14 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { if mi_unlikely(!_mi_is_power_of_two(block_size)) { adjust = diff % block_size; } + if (poffset!=NULL) { *poffset = adjust; } return (mi_block_t*)((uintptr_t)p - adjust); } +mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { + return mi_page_ptr_unalign_ex(page,p,NULL); +} + static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) { mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer #if MI_SECURE > 0 @@ -124,40 +125,56 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, con #endif } -// forward declaration for a MI_GUARDED build -#if MI_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration -static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { - if (mi_block_ptr_is_guarded(block, p)) { - mi_block_unguard(page, block, p); - return true; +static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept { + MI_UNUSED(was_guarded); + if mi_likely(!mi_page_has_interior_pointers(page)) { + return mi_validate_block_from_ptr(page,p); } else { - return false; + size_t offset; + mi_block_t* const block = mi_page_ptr_unalign_ex(page,p,&offset); + #if MI_GUARDED || MI_PROFILE + if (offset >= sizeof(mi_block_t)) { + #if MI_PROFILE + if (block->next == MI_BLOCK_TAG_PROFILED) { + _mi_page_profile_on_free(page,block,p); + } + else + #endif + #if MI_GUARDED + if (block->next == MI_BLOCK_TAG_GUARDED) { + _mi_page_block_unguard(page, block, p); + *was_guarded = true; + } + else + #endif + { /* nothing */ } + } + #endif + return block; } } -#else -static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { - MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); - return false; -} -#endif - // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */); + bool was_guarded = false; + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + // mi_block_check_profiled(page,block,p); + // const bool was_guarded = mi_block_check_unguard(page, block, p); + mi_free_block_local(page, block, was_guarded, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) -static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_collect) mi_attr_noexcept { +static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_mt(page, block, was_guarded, allow_collect); + bool was_guarded = false; + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + // mi_block_check_profiled(page,block,p); + // const bool was_guarded = mi_block_check_unguard(page, block, p); + mi_free_block_mt(page, block, was_guarded, allow_reclaim); } // generic free (for runtime integration) @@ -173,7 +190,7 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha { MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG - if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { + if mi_unlikely(!_mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return false; } @@ -181,21 +198,26 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha mi_page_t* page; #if MI_PAGE_META_SMALL_IS_ALIGNED - if (free_small) { page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); } + if (free_small) { + const uintptr_t up = _mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc + if mi_unlikely(up==0 && check_p_for_null) { return false; } + page = (mi_page_t*)up; + } else #endif - #if MI_PAGE_META_IS_ALIGNED - { page = _mi_aligned_ptr_page0(p); } - #else - { page = _mi_ptr_page(p); } - #endif - - if mi_unlikely(check_p_for_null && page==NULL) { - #if MI_DEBUG - if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } - #endif - return false; - } + { + #if MI_PAGE_META_IS_ALIGNED + { page = _mi_aligned_ptr_page0(p); } + #else + { page = _mi_ptr_page(p); } + #endif + if mi_unlikely(page==NULL && check_p_for_null) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + return false; + } + } #if MI_DEBUG mi_page_t* const cpage = _mi_checked_ptr_page(p); if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } @@ -233,7 +255,7 @@ static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim) { mi_assert_internal(p!=NULL && page!=NULL); if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } @@ -243,7 +265,7 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); + mi_free_block_local(page, block, false /* was guarded */, false /* no need to check if the page is full */); } else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` // page is local, but is full or contains (inner) aligned blocks; use generic path @@ -253,25 +275,25 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` // blocks are aligned (and not a full page); push on the thread_free list mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); + mi_free_block_mt(page,block,false /* was_guarded */, allow_reclaim); } else { // page is full or contains (inner) aligned blocks; use generic multi-thread path - mi_free_generic_mt(page, p, allow_collect); + mi_free_generic_mt(page, p, allow_reclaim); } } void mi_free(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { - mi_free_nonnull(p, page, pblock_size, true /* allow collect? */); + mi_free_nonnull(p, page, pblock_size, true /* allow reclaim? */); } else { if (pblock_size!=NULL) { *pblock_size = 0; } @@ -281,7 +303,7 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { void mi_free_small(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } @@ -289,15 +311,16 @@ void mi_free_small_nonnull(void* p) mi_attr_noexcept { mi_assert(p!=NULL); mi_page_t* page; if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } // Free a pointer that is potentially allocated in a different sub-process void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { - mi_free_nonnull(p, page, NULL, false /* allow collect? */); + if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { + // const bool allow_reclaim = (_mi_subproc() == mi_page_subproc(page)); + mi_free_nonnull(p, page, NULL, false /* allow reclaim */); } } @@ -358,7 +381,7 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { bool mi_cfree(void* p) mi_attr_noexcept { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); return true; } else { @@ -483,12 +506,12 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // We freed a block in an abandoned page (that was not owned). Try to collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); - // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses + // mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { @@ -496,14 +519,14 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted). // however, if the page became completely free, the used count is guaranteed to be 0.) mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used. - _mi_page_free_collect_partly(page, mt_free); + mt_free = _mi_page_free_collect_partly(page, mt_free); } else { // for larger blocks we use the regular collect _mi_page_free_collect(page,false /* no force */); mt_free = NULL; // expected page->xthread_free value after collection } - const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free); + const long reclaim_on_free = allow_reclaim && _mi_option_get_fast(mi_option_page_reclaim_on_free); #if MI_DEBUG > 1 if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); } @@ -526,8 +549,8 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // ------------------------------------------------------ // Bytes available in a block -static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept { - const mi_block_t* block = _mi_page_ptr_unalign(page, p); +static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p, const mi_block_t* block) mi_attr_noexcept { + // const mi_block_t* block = _mi_page_ptr_unalign(page, p); const bool is_guarded = mi_block_ptr_is_guarded(block,p); const size_t size = mi_page_usable_size_of(page, block, is_guarded); mi_assert_internal((void*)p >= (void*)block); @@ -540,13 +563,13 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { if mi_unlikely(page==NULL) return 0; mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); - if mi_likely(!mi_page_has_interior_pointers(page)) { - const mi_block_t* block = mi_validate_block_from_ptr(page,p); + const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign + if mi_likely((const void*)block==page) { return mi_page_usable_size_of(page, block, false /* is guarded */); } else { // split out to separate routine for improved code generation - return mi_page_usable_aligned_size_of(page, p); + return mi_page_usable_aligned_size_of(page, p, block); } } @@ -595,7 +618,7 @@ static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + return (_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } @@ -658,6 +681,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl // contain the pointer for the delayed list, then shrink the padding (by decreasing delta) // so it will later not trigger an overflow error in `mi_free_block`. void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + if (mi_block_ptr_is_guarded(block,block + 1)) return; size_t bsize; size_t delta; bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); @@ -749,7 +773,7 @@ mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, co #endif // only maintain stats for smaller objects if requested -#if (MI_STAT>0) +#if 0 && (MI_STATS>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(block); mi_theap_t* theap = _mi_theap_default(); @@ -765,13 +789,13 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { } const size_t bsize = mi_page_usable_block_size(page); - // #if (MI_STAT>1) + // #if (MI_STATS>=2) // const size_t usize = mi_page_usable_size_of(page, block); // mi_theap_stat_decrease(theap, malloc_requested, usize); // #endif if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { mi_theap_stat_decrease(theap, malloc_normal, bsize); - #if (MI_STAT > 1) + #if (MI_STATS > 1) mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1); #endif } @@ -791,34 +815,3 @@ void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { #endif -// Remove guard page when building with MI_GUARDED -#if MI_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { - MI_UNUSED(p); - mi_assert_internal(mi_block_ptr_is_guarded(block, p)); - mi_assert_internal(mi_page_has_interior_pointers(page)); - mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); - mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); - - const size_t bsize = mi_page_block_size(page); - const size_t psize = _mi_os_page_size(); - mi_assert_internal(bsize > psize); - mi_assert_internal(!page->memid.is_pinned); - void* gpage = (uint8_t*)block + bsize - psize; - mi_assert_internal(_mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); -} - -// unguard a whole page (called from `mi_heap_destroy`) -void _mi_page_unguard_all(mi_page_t* page) { - if mi_likely(!mi_page_has_interior_pointers(page)) return; - uint8_t* const start = mi_page_start(page); - const size_t psize = mi_page_committed(page); - _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded -} -#else -void _mi_page_unguard_all(mi_page_t* page) { - MI_UNUSED(page); - // nothing to do -} -#endif diff --git a/src/heap.c b/src/heap.c index 3e7611b5e..d7f1db132 100644 --- a/src/heap.c +++ b/src/heap.c @@ -87,7 +87,7 @@ static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_he // get (and possibly create) the theap belonging to a heap -mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) +mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) { mi_assert_internal(heap->theap != 0); mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); @@ -107,6 +107,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count); heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id); heap->numa_node = -1; // no initial affinity + heap->profiler = mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler); mi_stats_header_init(&heap->stats); mi_lock_init(&heap->theaps_lock); mi_lock_init(&heap->os_abandoned_pages_lock); @@ -182,6 +183,12 @@ static void mi_heap_free_theaps(mi_heap_t* heap) { theap = next; } } + + // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap (through delete pages -> page_update_stats)) + if (!_mi_is_process_heap_main(heap)) { + _mi_thread_local_free(heap->theap); + heap->theap = 0; + } } // free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed) @@ -221,7 +228,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { mi_lock_done(&heap->os_abandoned_pages_lock); mi_lock_done(&heap->arena_pages_lock); if (!_mi_is_process_heap_main(heap)) { - _mi_thread_local_free(heap->theap); + // _mi_thread_local_free(heap->theap); _mi_free_subproc_safe(heap); } } @@ -287,7 +294,7 @@ bool mi_check_owned(const void* p) { bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept { if (p==NULL) return false; const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer - if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false; + if (page==NULL || mi_page_used(page)==page->capacity || page->capacity < page->reserved) return false; // If the page is the head of the queue, it is currently being used for // allocations; we skip it to avoid immediate thrashing. if (page->prev == NULL) return false; @@ -300,5 +307,5 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc // check utilization if (page->capacity==0) return false; if (perc_threshold>=100) return true; - return (perc_threshold >= ((100UL*page->used) / page->capacity)); + return (perc_threshold >= ((100UL*mi_page_used(page)) / page->capacity)); } diff --git a/src/init.c b/src/init.c index 68e94aaa2..30445a58c 100644 --- a/src/init.c +++ b/src/init.c @@ -19,7 +19,11 @@ static const mi_page_t mi_page_empty = { #endif MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free - 0, // used + {0}, // xused + #if MI_SIZE_SIZE < 8 + 0, // xlast_used + 0, // xlast_alloc + #endif NULL, // local_free 0, // block_size 0, // page_offset @@ -39,8 +43,6 @@ static const mi_page_t mi_page_empty = { #else { 0 }, // key #endif - // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 - // { 0 }, // padding #endif }; @@ -54,7 +56,7 @@ static const mi_page_t mi_page_empty = { #error define initializer for direct pages #endif -#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) +#if (MI_PADDING>0) && (MI_SIZE_SIZE >= 8) #define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #elif (MI_PADDING>0) #define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } @@ -123,6 +125,14 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount + 0, // full page retain + false, // allow reclaim + true, // allow abandon + true, // is_detached + ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) + 0, 0, // sample rate, requested + 0, 0, // profile rate, countdown + 0, 0, 0, 0, // guarded rate, countdown, min, max 0, // heartbeat { {0}, {0}, 0, true }, // random 0, // page count @@ -131,13 +141,6 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, 0, // generic count NULL, NULL, // tnext, tprev NULL, NULL, // hnext, hprev - 0, // full page retain - false, // allow reclaim - true, // allow abandon - true, // is_detached - #if MI_GUARDED - 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) - #endif MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, MI_STATS_NULL, // stats @@ -201,6 +204,8 @@ static void mi_heap_main_init_once(void) { _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads mi_process_theap_meta.page_full_retain = 2; + mi_process_theap_meta.sample_rate = 0; // no sampling for meta data + mi_process_theap_meta.sample_countdown = 0; subproc_main->theap_meta = &mi_process_theap_meta; // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` @@ -441,6 +446,11 @@ static void mi_thread_theaps_done(mi_tld_t* tld) static void mi_process_setup_auto_thread_done(void) { mi_atomic_do_once { _mi_prim_thread_init_auto_done(); + mi_theap_t* theap = _mi_theap_default(); + mi_assert_internal(mi_theap_is_initialized(theap)); + if (mi_theap_is_initialized(theap)) { + _mi_theap_default_set(theap); + } } } @@ -449,7 +459,7 @@ void mi_thread_done(void) mi_attr_noexcept { } void _mi_thread_done(mi_theap_t* _theap_main) -{ +{ // NULL can be passed on some platforms if (_theap_main==NULL) { _theap_main = _mi_theap_default(); @@ -506,7 +516,6 @@ void _mi_auto_process_init(void) { os_preloading = false; mi_process_init(); - mi_process_setup_auto_thread_done(); _mi_options_post_init(); // now we can print to stderr if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); @@ -551,6 +560,7 @@ static void mi_process_init_once(void) { // the following can potentially allocate (on freeBSD for pthread keys) _mi_tls_slots_init(); // pthread key create _mi_thread_locals_init(); // pthread key create + mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap _mi_process_is_initialized = true; #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) @@ -627,6 +637,7 @@ static void mi_process_done_once(void) { _mi_thread_locals_done(); if (subproc_main->heap_main != NULL) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + mi_theap_collect(subproc_main->theap_meta, false /* force */); // update stats of all pages in theap_meta _mi_theap_merge_stats(subproc_main->theap_meta); _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free mi_heap_stats_merge_to_subproc(subproc_main->heap_main); diff --git a/src/libc.c b/src/libc.c index 17dd1d1dc..8f211230d 100644 --- a/src/libc.c +++ b/src/libc.c @@ -160,8 +160,6 @@ mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruc // -------------------------------------------------------- // Detect CPU features // -------------------------------------------------------- -mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast -mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast mi_decl_cache_align bool _mi_cpu_has_popcnt = false; #if (MI_ARCH_X64 || MI_ARCH_X86) @@ -192,39 +190,15 @@ static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { #endif void _mi_detect_cpu_features(void) { - // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) - // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) - // FSRS for fast short rep stosb - bool amd = false; - bool fsrm = false; - // bool erms = false; - bool fsrs = false; uint32_t cpu_info[4]; - if (mi_cpuid(cpu_info, 0, 0)) { - amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) - } - if (mi_cpuid(cpu_info, 7, 0)) { - fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see - // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see - } - if (mi_cpuid(cpu_info, 7, 1)) { - fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see - } if (mi_cpuid(cpu_info, 1, 0)) { _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see } - - if (fsrm) { - _mi_cpu_movsb_max = 127; - } - if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 - _mi_cpu_stosb_max = 127; - } } #else void _mi_detect_cpu_features(void) { - #if MI_ARCH_ARM64 + #if MI_ARCH_ARM64 || defined(__riscv_zbb) || defined(__riscv_b) _mi_cpu_has_popcnt = true; #endif } diff --git a/src/options.c b/src/options.c index da0d6669c..eaa2119de 100644 --- a/src/options.c +++ b/src/options.c @@ -49,7 +49,7 @@ int mi_version(void) { // in KiB #ifndef MI_DEFAULT_ARENA_RESERVE - #if (MI_INTPTR_SIZE>4) + #if (MI_SIZE_SIZE>4) #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L #else #define MI_DEFAULT_ARENA_RESERVE 128L*1024L @@ -77,10 +77,10 @@ int mi_version(void) { #endif #ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE -#if MI_GUARDED && !MI_DEBUG -#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000 +#if MI_GUARDED && MI_DEBUG +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 /* MI_MiB */ #else -#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 #endif #endif @@ -109,6 +109,10 @@ int mi_version(void) { #endif #endif +#ifndef MI_DEFAULT_COLLECT_MERGES_STATS +#define MI_DEFAULT_COLLECT_MERGES_STATS 1 +#endif + // Static options static mi_option_desc_t mi_options[_mi_option_last] = { @@ -176,6 +180,8 @@ static mi_option_desc_t mi_options[_mi_option_last] = { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation + { MI_DEFAULT_COLLECT_MERGES_STATS, + MI_OPTION_UNINIT, MI_OPTION(collect_merges_stats) }, // on each theap collect, stats are merged with the parent heap }; static void mi_option_init(mi_option_desc_t* desc); @@ -240,30 +246,34 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : "")); } - // show build configuration - _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); - _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); - _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); - #if MI_GUARDED - _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled"); - #endif - #if MI_TSAN - _mi_fprintf(out, arg, "thread santizer enabled\n"); - #endif + // show build configuration #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED - _mi_fprintf(out, arg, "free: (small) aligned, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t)); #elif MI_PAGE_META_IS_ALIGNED - _mi_fprintf(out, arg, "free: aligned, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : aligned, page size: %zu\n", sizeof(mi_page_t)); #elif MI_PAGE_META_SMALL_IS_ALIGNED - _mi_fprintf(out, arg, "free: small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); #elif MI_FREE_IS_CHECKED - _mi_fprintf(out, arg, "free: checked, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : checked, page size: %zu\n", sizeof(mi_page_t)); #else - _mi_fprintf(out, arg, "free: pagemap, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : pagemap, page size: %zu\n", sizeof(mi_page_t)); #endif #if MI_ENCODE_FREELIST - _mi_fprintf(out, arg, "free lists: encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + _mi_fprintf(out, arg, "free lists : encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + #endif + #if MI_GUARDED + _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate)); + #endif + #if MI_PROFILE + _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled"); + #endif + #if MI_TSAN + _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif + _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); + _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); + _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); + _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise"))); } mi_decl_export void mi_options_print(void) mi_attr_noexcept { diff --git a/src/os.c b/src/os.c index a727cc844..15111cf94 100644 --- a/src/os.c +++ b/src/os.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file Initialization. ----------------------------------------------------------- */ #ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB -#if MI_INTPTR_SIZE < 8 +#if MI_SIZE_SIZE < 8 #define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB #else #define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB @@ -119,7 +119,7 @@ bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero // On systems with enough virtual address bits, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address // space (64TiB) we use this technique. (but see issue #939) -#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) +#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) // Return a `try_alignment` aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be @@ -361,7 +361,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); - #if MI_INTPTR_SIZE >= 8 + #if MI_SIZE_SIZE >= 8 const bool try_direct_alloc = true; #else // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. @@ -730,7 +730,7 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`) #define MI_HUGE_OS_PAGE_SIZE (MI_GiB) -#if (MI_INTPTR_SIZE >= 8) +#if (MI_SIZE_SIZE >= 8) // To ensure proper alignment, use our own area for huge OS pages static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 diff --git a/src/page-queue.c b/src/page-queue.c index c10c09257..c8e9b5425 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -21,11 +21,11 @@ terms of the MIT license. A copy of the license can be found in the file Minimal alignment in machine words (i.e. `sizeof(void*)`) ----------------------------------------------------------- */ -#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) +#if (MI_MAX_ALIGN_SIZE > 4*MI_SIZE_SIZE) #error "define alignment for more than 4x word size for this platform" -#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) +#elif (MI_MAX_ALIGN_SIZE > 2*MI_SIZE_SIZE) #define MI_ALIGN4W // 4 machine words minimal alignment -#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) +#elif (MI_MAX_ALIGN_SIZE > MI_SIZE_SIZE) #define MI_ALIGN2W // 2 machine words minimal alignment #else // ok, default alignment is 1 word @@ -62,7 +62,7 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. static size_t mi_bin(size_t size) { - size_t wsize = _mi_wsize_from_size(size); + size_t wsize = mi_wsize_from_size(size); #if defined(MI_ALIGN4W) if mi_likely(wsize <= 4) { return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes @@ -152,10 +152,10 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { for (mi_page_t* page = pq->first; page != NULL; page = page->next) { mi_assert_internal(page->prev == prev); if (mi_page_is_in_full(page)) { - mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); + mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); } else if (mi_page_is_huge(page)) { - mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); + mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); } else { mi_assert_internal(mi_page_block_size(page) == pq->block_size); @@ -215,7 +215,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ if (pq->first == NULL) page = _mi_page_empty_get(); // find index in the right direct page array - const size_t idx = _mi_wsize_from_size(size); + const size_t idx = mi_wsize_from_size(size); mi_page_t** const pages_free = theap->pages_free_direct; if (pages_free[idx] == page) return; // already set @@ -232,7 +232,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) { prev--; } - start = 1 + _mi_wsize_from_size(prev->block_size); + start = 1 + mi_wsize_from_size(prev->block_size); if (start > idx) start = idx; } @@ -341,6 +341,15 @@ static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queu mi_assert_internal(queue->first == page); } +static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->last == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push_at_end(theap, queue, page); + mi_assert_internal(queue->last == page); +} + static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(from->count >= 1); diff --git a/src/page.c b/src/page.c index 02ca72044..aa669cdd0 100644 --- a/src/page.c +++ b/src/page.c @@ -83,7 +83,7 @@ static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_block_size(page) > 0); - mi_assert_internal(page->used <= page->capacity); + mi_assert_internal(mi_page_used(page) <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); mi_assert_internal(page->heap!=NULL); @@ -114,8 +114,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) { #endif size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free); - mi_assert_internal(page->used + free_count == page->capacity); + mi_assert_internal(mi_page_used(page) + free_count == page->capacity); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); return true; } @@ -138,10 +139,180 @@ bool _mi_page_is_valid(mi_page_t* page) { // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq)); } } + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); return true; } #endif +#if MI_STATS || MI_SAMPLE +// Gets the theap belonging to a page. +static mi_theap_t* mi_theap_of_page(mi_page_t* page) { + mi_theap_t* theap = page->theap; + if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { + theap = _mi_page_associated_theap_peek(page); + } + return theap; +} +#endif + +#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc.c:mi_page_alloc_zero` */ +static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) +{ + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(theap!=NULL); + if (theap->sample_rate==0) return; + + const size_t last_alloc = mi_page_last_alloc(page); + if (alloc_count <= last_alloc) return; + + // update countdown + const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE); + const size_t alloc_diff = alloc_count - last_alloc; + const uint64_t requested = (uint64_t)alloc_diff * (uint64_t)(bsize - MI_PADDING_SIZE); + if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) { + theap->sample_countdown -= (size_t)requested; + } + else { + theap->sample_requested += (requested - theap->sample_countdown); + theap->sample_countdown = 0; + } +} +#endif + +#if MI_STATS +// Merge stats from the page into the corresponding theap or heap. +static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) { + // get heap (as the theap might be NULL) + mi_heap_t* const heap = mi_page_heap(page); + mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id)); + mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); + mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); + + // allocation sizes + const size_t bsize = mi_page_usable_block_size(page); + const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize; + const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize; + #if MI_STATS==1 + const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE); + #endif + mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats + mi_assert_internal(freed <= INT64_MAX); + + // adjust stats + if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { + const size_t bin = _mi_bin(bsize); + // allocations + if (alloc_count > 0) { + mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_normal, allocated); + mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); + #endif + } + // frees + if (free_count > 0) { + mi_theapx_stat_decrease(heap, theap, malloc_normal, freed); + mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); + } + } + else { + // allocations + mi_assert_internal(alloc_count<=1); + mi_assert_internal(free_count<=1); + if (alloc_count > 0) { + mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_huge, allocated); + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); + #endif + } + // frees + if (free_count > 0) { + mi_theapx_stat_decrease(heap, theap, malloc_huge, freed); + } + } +} + +// Update stats for a page +static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + // get stat counts + const size_t used = mi_page_used(page); + const size_t alloc_count = mi_page_alloc_count(page); + const size_t last_used = mi_page_last_used(page); + mi_assert_internal(last_used + alloc_count >= used); + mi_assert_internal(used <= UINT16_MAX); + const size_t free_count = last_used + alloc_count - used; + + #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero` + if (theap!=NULL) { mi_theap_adjust_sample_countdown(theap,page,alloc_count); } + #endif + + // update stats? + if (alloc_count + free_count == 0) { + mi_assert_internal(last_used == used); + return; + } + + // reset the `alloc_count` (and `last_alloc`), and set `last_used` to `used` + #if MI_SIZE_SIZE >= 8 + page->xused.used_alloc = (used << 32) | used; + #else + page->xused.used_alloc = used; + page->xlast_used = (uint16_t)used; + page->xlast_alloc = 0; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + mi_theap_page_merge_stats(theap, page, alloc_count, free_count); +} + +void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pages etc. + mi_theap_page_update_stats(mi_theap_of_page(page),page); +} + +#else +void _mi_page_update_stats(mi_page_t* page) { + MI_UNUSED(page); +} +#endif + +static void mi_page_update_sample_countdown(mi_page_t* page) +{ + // adjust count down + const size_t alloc_count = mi_page_alloc_count(page); + if (alloc_count==0) { + return; + } + else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the counter is reset + _mi_page_update_stats(page); + } + #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc.c:mi_page_alloc_zero_ex` + else { + mi_theap_t* theap = mi_theap_of_page(page); + if (theap==NULL) return; + mi_theap_adjust_sample_countdown(theap,page,alloc_count); + // update last_alloc to alloc_count + mi_assert_internal(alloc_count <= UINT16_MAX); + #if MI_SIZE_SIZE >= 8 + page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16)); + #else + page->xlast_alloc = (uint16_t)alloc_count; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + } + #endif +} + /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists @@ -167,7 +338,7 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) return; // the thread-free items cannot be freed } // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list) - else if mi_unlikely(count > page->used) { + else if mi_unlikely(count > mi_page_used(page)) { _mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n"); return; // the thread-free items cannot be freed } @@ -178,8 +349,8 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) // update counts now mi_assert_internal(count <= UINT16_MAX); - mi_assert_internal(page->used >= (uint16_t)count); - page->used = page->used - (uint16_t)count; + mi_assert_internal(mi_page_used(page) >= count); + page->xused.used_alloc -= count; // page->used = page->used - (uint16_t)count; } // Collect the local `thread_free` list using an atomic exchange. @@ -200,6 +371,7 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_page_thread_collect_to_local(page, head); } + // returns `true` if after collection `mi_page_immediate_available` is true. static inline bool mi_page_free_quick_collect(mi_page_t* page) { if mi_likely(page->free != NULL) return true; @@ -207,7 +379,8 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { // move local_free to free page->free = page->local_free; page->local_free = NULL; - page->free_is_zero = false; + page->free_is_zero = false; + mi_page_update_sample_countdown(page); return true; } @@ -237,19 +410,20 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->local_free = NULL; page->free_is_zero = false; } - } - + mi_page_update_sample_countdown(page); + } mi_assert_internal(!force || page->local_free == NULL); } // Collect elements in the thread-free list starting at `head`. This is an optimized // version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. +// returns a possibly updated expected value for the thread_free pointer. // // `head` must be in the `xthread_free` list. It will not collect `head` itself // so the `used` count is not fully updated in general. However, if the `head` is // the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true). -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { - if (head == NULL) return; +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { + if (head == NULL) return NULL; mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops) if (next != NULL) { mi_block_set_next(page, head, NULL); @@ -258,13 +432,18 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - } + mi_page_update_sample_countdown(page); + } } - if (page->used == 1) { + if (mi_page_used(page) == 1) { // all elements are free'd since we skipped the `head` element itself mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head); mi_assert_internal(mi_block_next(page,head) == NULL); _mi_page_free_collect(page, false); // collect the final element + return NULL; + } + else { + return head; } } @@ -283,6 +462,7 @@ void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) mi_page_set_theap(page,theap); _mi_page_free_collect(page, false); // ensure used count is up to date + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); mi_page_queue_push_at_end(theap, pq, page); mi_assert_expensive(_mi_page_is_valid(page)); @@ -385,6 +565,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed) mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page); _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set + _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages } } @@ -558,15 +739,15 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* // and initialize the free list by randomly threading through them // set up first element - const uintptr_t r = _mi_theap_random_next(theap); + const size_t r = _mi_theap_random_next(theap); size_t current = r % slice_count; counts[current]--; mi_block_t* const free_start = blocks[current]; // and iterate through the rest; use `random_shuffle` for performance - uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0 + size_t rnd = _mi_random_shuffle(r|1); // ensure not 0 for (size_t i = 1; i < extend; i++) { - // call random_shuffle only every INTPTR_SIZE rounds - const size_t round = i%MI_INTPTR_SIZE; + // call random_shuffle only every SIZE_SIZE rounds + const size_t round = i%MI_SIZE_SIZE; if (round == 0) rnd = _mi_random_shuffle(rnd); // select a random next slice index size_t next = ((rnd >> 8*round) & (slice_count-1)); @@ -639,10 +820,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { size_t page_size; //uint8_t* page_start = mi_page_area(page, &page_size); - #if MI_STAT>0 mi_theap_stat_counter_increase(theap, pages_extended, 1); - #endif - + // calculate the extend count const size_t bsize = mi_page_block_size(page); size_t extend = (size_t)page->reserved - page->capacity; @@ -698,9 +877,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { } // enable the new free list page->capacity += (uint16_t)extend; - #if MI_STAT>0 mi_theap_stat_increase(theap, page_committed, extend * bsize); - #endif mi_assert_expensive(mi_page_is_valid_init(page)); return true; } @@ -736,7 +913,8 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->theap == mi_page_theap(page)); mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); - mi_assert_internal(page->used == 0); + mi_assert_internal(mi_page_used(page) == 0); + mi_assert_internal(page->xused.used_alloc == 0); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(page->xthread_free == 1); mi_assert_internal(page->next == NULL); @@ -771,8 +949,9 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages mi_page_t* page_candidate = NULL; // a page with free space mi_page_t* page = pq->first; + mi_page_t* const last = pq->last; - while (page != NULL) + while (page!=NULL) { mi_page_t* next = page->next; // remember next (as this page can move to another queue) count++; @@ -796,6 +975,10 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); } + else if (page!=last && pq->last!=page) { + // avoid revisiting this page for a while + mi_page_queue_move_to_back(theap, pq, page); + } } else { // the page has free space, make it a candidate @@ -809,7 +992,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, page_candidate = page; } // prefer to reuse fuller pages (in the hope the less used page gets freed) - else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { + else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { page_candidate = page; } // if we find a non-expandable candidate, or searched for N pages, return with the best candidate @@ -831,8 +1014,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); #endif - - page = next; + if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back + else { page = next; } } // for each page mi_theap_stat_counter_increase(theap, page_searches, count); @@ -939,8 +1122,8 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page mi_assert_internal(mi_page_is_abandoned(page)); mi_page_set_theap(page, NULL); #endif - mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); - mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); + // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); + // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); } return page; } @@ -949,7 +1132,7 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ? _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); return NULL; } @@ -1007,35 +1190,54 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex /* ----------------------------------------------------------- Admin ----------------------------------------------------------- */ +static mi_theap_t* mi_theap_init(mi_theap_t* theap) { + if mi_likely(mi_theap_is_initialized(theap)) return theap; + if (theap==&_mi_theap_empty_wrong) { + // we were unable to allocate a theap for a first-class heap + return NULL; + } + // otherwise we initialize the thread and its default theap + theap = _mi_thread_init(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } + mi_assert_internal(mi_theap_is_initialized(theap)); + return theap; +} static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) { - if mi_unlikely(!mi_theap_is_initialized(theap)) { - if (theap==&_mi_theap_empty_wrong) { - // we were unable to allocate a theap for a first-class heap - return NULL; - } - // otherwise we initialize the thread and its default theap - theap = _mi_thread_init(); - if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } - } + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; mi_assert_internal(mi_theap_is_initialized(theap)); // do administrative tasks every N generic mallocs if mi_unlikely(theap->generic_count >= 1000) { theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; - + + // check if the profiler is enabled + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); + if (prof_enabled && theap->profile_sample_rate==0) { + _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling + } + else if (!prof_enabled && theap->profile_sample_rate!=0) { + _mi_theap_set_profile_sample_rate(theap,0); // stop profiling + } + // do a full theap collect every once in a while (10000 by default) const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); if (theap->generic_collect_count >= generic_collect) { theap->generic_collect_count = 0; mi_theap_collect(theap, false /* force? */); } + // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling + // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? + // } else { // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines - _mi_theap_collect_retired(theap, false); // free retired pages + _mi_theap_collect_retired(theap, false); // free retired pages } } return theap; @@ -1051,6 +1253,35 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size theap = mi_malloc_generic_admin(theap); if (theap==NULL) return NULL; + // check allocation size + const size_t req_size = size - MI_PADDING_SIZE; + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return NULL; + } + + // take a sample? + bool sample_countdown_is_adjusted = false; + if mi_unlikely(mi_theap_should_sample(theap,req_size)) { + if (huge_alignment==0 && theap->sample_rate!=0) { + return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); + } + mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap + mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); + if (theap->sample_rate==0) { + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter + } + #if MI_SAMPLE==2 + else { + // huge_alignment!=0 so we need to adjust the countdown + mi_assert_internal(huge_alignment!=0); + mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size); + sample_countdown_is_adjusted = true; + theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again + } + #endif + } + // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(theap, size, huge_alignment); if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more @@ -1059,9 +1290,9 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size } if mi_unlikely(page == NULL) { // out of memory - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (sample_countdown_is_adjusted) { theap->sample_countdown -= req_size; } // unadjust again if we failed to find a page _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); - return NULL; + return NULL; } mi_assert_internal(mi_page_immediate_available(page)); @@ -1069,15 +1300,19 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero) if (ppage!=NULL) { *ppage = page; } void* const p = _mi_page_malloc_zero(theap,page,size,zero); mi_assert_internal(p != NULL); - + mi_page_update_sample_countdown(page); + // move full pages to the full queue - if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) { - mi_page_to_full(page, mi_page_queue_of(page)); - } + // this will also call _mi_page_update_stats for huge pages + if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { + if (mi_page_is_full(page)) { + mi_page_to_full(page, mi_page_queue_of(page)); + } + } return p; } @@ -1099,19 +1334,38 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm // fast path objects that fit in a small page if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if (req_size < MI_SMALL_MAX_OBJ_SIZE) { - mi_page_queue_t* pq = mi_page_queue(theap, size); - mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); - page = mi_page_queue_find_free(theap,pq); - // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); - if (page!=NULL) { - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(mi_page_immediate_available(page)); - return _mi_page_malloc_zero(theap,page,size,zero); + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) + { + #if MI_SAMPLE + if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample + #endif + { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero + return _mi_page_malloc_zero(theap,page,size,zero); + } } } } // otherwise fallback - return mi_malloc_generic_fallback(theap,size,zero, huge_alignment,ppage); + return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); +} + +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; + const size_t sample_rate = theap->sample_rate; + const size_t sample_countdown = theap->sample_countdown; + theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; + void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + theap->sample_rate = sample_rate; + theap->sample_countdown = sample_countdown; + return p; } diff --git a/src/prim/prim-tls.c b/src/prim/prim-tls.c index 8c8b0c71f..1b79321a0 100644 --- a/src/prim/prim-tls.c +++ b/src/prim/prim-tls.c @@ -46,9 +46,9 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { // If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) // See for the offsets. #if MI_SIZE_SIZE==4 -#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) +#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE) #else -#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) +#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE) #endif #define MI_TLS_DIRECT_SLOTS (64) #define MI_TLS_EXPANSION_SLOTS (1024) diff --git a/src/prim/prim.c b/src/prim/prim.c index 7f31c994d..3d3d0c2e8 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -29,9 +29,8 @@ terms of the MIT license. A copy of the license can be found in the file // Generic process initialization #if !defined(MI_PRIM_HAS_PROCESS_ATTACH) #if defined(__GNUC__) || defined(__clang__) - // gcc,clang: use the constructor/destructor attribute - // which for both seem to run before regular constructors/destructors - #if defined(__clang__) + // gcc,clang: use the constructor/destructor attribute with the highest user priority (<=100 is reserved for the compiler itself) + #if defined(__clang__) || (__GNUC__ >= 5 && !defined(__APPLE__)) // gcc has support since 4.2 but not on macOS (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119435) #define mi_attr_constructor __attribute__((constructor(101))) #define mi_attr_destructor __attribute__((destructor(101))) #else diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 1838901d5..25c2adea0 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -339,7 +339,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen // fall back to regular mmap } #endif - #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + #if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED) // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment, size); @@ -612,7 +612,7 @@ int _mi_prim_protect(void* start, size_t size, bool protect) { // Huge page allocation //--------------------------------------------- -#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) +#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) #ifndef MPOL_PREFERRED #define MPOL_PREFERRED 1 @@ -633,12 +633,12 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo bool is_large = true; *is_zero = true; *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large); - if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_INTPTR_SIZE - 1)) { // at most 63 nodes + if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { // at most 63 nodes unsigned long numa_mask = (1UL << numa_node); // todo: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: - long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0); if (err != 0) { err = errno; _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err); diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 0dfc42202..222fa6303 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -278,7 +278,7 @@ int _mi_prim_free(void* addr, size_t size ) { //--------------------------------------------- static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) { - #if (MI_INTPTR_SIZE >= 8) + #if (MI_SIZE_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment,size); diff --git a/src/random.c b/src/random.c index 464627acf..3c012adf4 100644 --- a/src/random.c +++ b/src/random.c @@ -139,14 +139,14 @@ void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); } -uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { +size_t _mi_random_next(mi_random_ctx_t* ctx) { mi_assert_internal(mi_random_is_initialized(ctx)); - uintptr_t r; + size_t r; do { - #if MI_INTPTR_SIZE <= 4 + #if MI_SIZE_SIZE <= 4 r = chacha_next32(ctx); - #elif MI_INTPTR_SIZE == 8 - r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); + #elif MI_SIZE_SIZE == 8 + r = (((size_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); #else # error "define mi_random_next for this platform" #endif @@ -160,12 +160,12 @@ To initialize a fresh random context. If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ -uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { - uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random +size_t _mi_os_random_weak(size_t extra_seed) { + size_t x = (size_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random x ^= _mi_prim_clock_now(); // and do a few randomization steps - uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; - for (uintptr_t i = 0; i < max || x==0; i++, x++) { + size_t max = ((x ^ (x >> 17)) & 0x0F) + 1; + for (size_t i = 0; i < max || x==0; i++, x++) { x = _mi_random_shuffle(x); } mi_assert_internal(x != 0); @@ -180,7 +180,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { #if !defined(__wasi__) if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } #endif - uintptr_t x = _mi_os_random_weak(0); + size_t x = _mi_os_random_weak(0); for (size_t i = 0; i < 32; i+=4, x++) { x = _mi_random_shuffle(x); key[i] = (uint8_t)(x); diff --git a/src/sample-guarded.c b/src/sample-guarded.c new file mode 100644 index 000000000..f7c1fe81e --- /dev/null +++ b/src/sample-guarded.c @@ -0,0 +1,173 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); + if (theap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_theap_random_next(theap); + } + theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } + else if (theap->guarded_sample_countdown > theap->guarded_sample_rate) { + theap->guarded_sample_countdown = theap->guarded_sample_rate; + } + _mi_theap_update_sample_rate(theap); +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + theap->guarded_size_min = min; + theap->guarded_size_max = (min > max ? min : max); +} + +void _mi_theap_guarded_init(mi_theap_t* theap) { + #if MI_GUARDED + mi_theap_guarded_set_sample_rate(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_theap_guarded_set_size_bound(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); + #else + mi_theap_guarded_set_sample_rate(theap,0,0); + #endif +} + + +// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { + // todo: we can still make padding work by moving it out of the guard page area + mi_page_t* const page = _mi_ptr_page(block); + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_GUARDED; + + // set guard page at the end of the block + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); + if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { + // should never happen + mi_free(block); + return NULL; + } + uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + // note: the alignment of the guard page relies on blocks being os_page_size aligned which + // is ensured in `mi_arena_page_alloc_fresh`. + mi_assert_internal(_mi_is_aligned(block, os_page_size)); + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); + if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { + const bool ok = _mi_os_protect(guard_page, os_page_size); + if mi_unlikely(!ok) { + _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); + } + } + else { + _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); + } + + // align pointer just in front of the guard page + size_t offset = block_size - os_page_size - obj_size; + mi_assert_internal(offset > sizeof(mi_block_t)); + if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { + // give up to place it right in front of the guard page if the offset is too large for unalignment + offset = MI_PAGE_MAX_OVERALLOC_ALIGN; + } + uint8_t* const p = (uint8_t*)block + offset; + mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); + if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } + mi_track_align(block, p, offset, obj_size); + mi_track_mem_defined(block, sizeof(mi_block_t)); + return p; +} + +// Allocate a block with a guard page behind it. +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // allocate multiple of page size ending in a guard page + // ensure minimal alignment requirement? + if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow + _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); + return NULL; + } + const size_t os_page_size = _mi_os_page_size(); + const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + // const size_t threshold = mi_theap_disable_profiler(theap); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); + // mi_theap_enable_profiler(theap,threshold); + if (block==NULL) return NULL; + size_t usable_size = 0; + void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); + if (p == NULL) return NULL; + if (zero) { + _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) + } + + // stats + mi_track_malloc(p, usable_size, zero); + if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } + mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); + #if MI_STATS + // adjust request stats to only count the allocated size of the block (and not the guard page) + mi_theap_stat_counter_decrease(theap, malloc_requested, req_size); + mi_theap_stat_counter_increase(theap, malloc_requested, size); + #endif + #if MI_DEBUG>3 + if (zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED && MI_DEBUG>=2 + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + if (size <= MI_SMALL_SIZE_MAX) { + mi_page_t* const page = _mi_ptr_page(p); + mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + } + #endif + return p; +} + + +// Remove guard page when building with MI_GUARDED +#if MI_GUARDED +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + // if (!mi_block_ptr_is_guarded(block,p)) return; + mi_assert_internal(mi_block_ptr_is_guarded(block, p)); + mi_assert_internal(mi_page_has_interior_pointers(page)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); + mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); + + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(!page->memid.is_pinned); + void* gpage = (uint8_t*)block + bsize - psize; + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); +} + +// unguard a whole page (called from `mi_heap_destroy`) +void _mi_page_unguard_all(mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page)) return; + uint8_t* const start = mi_page_start(page); + const size_t psize = mi_page_committed(page); + _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded +} +#else +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); +} +void _mi_page_unguard_all(mi_page_t* page) { + MI_UNUSED(page); +} +#endif diff --git a/src/sample-profile.c b/src/sample-profile.c new file mode 100644 index 000000000..f1b5db491 --- /dev/null +++ b/src/sample-profile.c @@ -0,0 +1,264 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +//---------------------------------------------------------------------------- +// General sampled allocation (called from `page.c:mi_malloc_generic_fallback`) +// This is for both guarded and profiled sampling +//---------------------------------------------------------------------------- + +size_t _mi_theap_update_sample_rate(mi_theap_t* theap) { + const size_t old_sample_rate = theap->sample_rate; + theap->sample_rate = theap->profile_sample_rate; + if (theap->sample_rate == 0 || (theap->guarded_sample_rate!=0 && theap->sample_rate > theap->guarded_sample_rate)) { + theap->sample_rate = theap->guarded_sample_rate; + } + if (theap->sample_countdown > theap->sample_rate) { + theap->sample_countdown = theap->sample_rate; // todo: adjust difference? + } + return old_sample_rate; +} + +// Called when the theap sample countdown <= req_size; should call guarded or profiled allocation. +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) + mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); + mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || + (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size)); + const size_t size = req_size + MI_PADDING_SIZE; + + // handle empty theap and disabled sampling + if (theap->sample_rate==0) { + if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time + } + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + } + + // update countdown and total accummulated requested bytes since the last sample + mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(req_size <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate > 0); + mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate >= theap->sample_countdown); + + const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; + mi_assert_internal(requested > 0); + mi_assert_internal(requested >= req_size); + theap->sample_countdown = theap->sample_rate; // reset sampling + + // update derived countdowns + mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0); + mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); + bool sample_profile = false; + bool sample_guarded = false; + if (theap->profile_sample_rate!=0) { + if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; } + } + if (theap->guarded_sample_rate!=0) { + if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; } + } + + // invoke callback? + if (sample_profile) { + theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown + theap->sample_requested = 0; // reset requested as we pass it to malloc_profiled + #if MI_PROFILE + return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage); + #endif + } + else if (sample_guarded) { + if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown + #if MI_GUARDED + return _mi_theap_malloc_guarded(theap,size,zero,ppage); + #endif + } + #if MI_GUARDED + else { + // failed size criteria, rewind the sample countdown so we sample asap again + // todo: can we do better here as this will cause many samples until it fits the size.. + theap->sample_countdown = 0; + } + #endif + } + // take generic path + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); +} + + + +//---------------------------------------------------------------------------- +// Util +//---------------------------------------------------------------------------- + +static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { + return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); +} + +static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc != NULL) { + return prof; + } + else { + return NULL; + } +} + +//---------------------------------------------------------------------------- +// Profile an allocation +//----------------------------------------------------------------------------- + +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + mi_assert_internal(theap!=NULL); + mi_assert_internal(size>=MI_PADDING_SIZE); + const size_t req_size = size - MI_PADDING_SIZE; + mi_assert_internal(req_size<=requested_since_last_sample); + mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); + if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } + mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof)); + + void* p = NULL; + size_t new_sample_rate = 0; + if (prof->on_free==NULL || prof->sample_data_size==0) { + // just allocate without profiler data + p = _mi_malloc_generic_no_sample(theap,size,zero,ppage); + if (p==NULL) { return p; } + if (prof->on_alloc!=NULL) { + // we are just allocation profiling (not heap profiling as on_free == NULL) + new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + } + } + else { + // overallocate a larger block to store the profiler data + // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] + const size_t sample_data_offset = sizeof(mi_block_t); + const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); + const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case + const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = user_offset + size; + mi_page_t* page = NULL; + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); + if (block==NULL) return NULL; + mi_assert_internal(page!=NULL); + if (ppage!=NULL) { *ppage = page; } + #if MI_PAGE_META_SMALL_IS_ALIGNED + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + // (that is why we need to limit the profiler_data_size as well) + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } + #endif + + // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_PROFILED; + p = (uint8_t*)block + user_offset; + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); + sample_data->user_data_size = sample_user_data_size; + + // and call the profiler on_alloc + if (prof->on_alloc!=NULL) { + new_sample_rate = (*prof->on_alloc)(prof, sample_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + } + } + if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { + _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + } + mi_theap_stat_counter_increase(theap,profile_samples,1); + return p; +} + +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { + mi_assert_internal(mi_block_ptr_is_sampled(block,p)); + + // get the heap and profiler + mi_heap_t* const heap = mi_page_heap(page); + if (heap==NULL) return; + mi_profiler_t* prof = mi_heap_profiler(heap); + if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; + + // call the on_free callback + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); + (*prof->on_free)(prof, sample_data, p, heap); +} + + +//---------------------------------------------------------------------------- +// Profiling API +//-----------------------------------------------------------------------------* + +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) { + const size_t old_sample_rate = theap->profile_sample_rate; + theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); + if (theap->profile_sample_countdown > theap->profile_sample_rate) { + theap->profile_sample_countdown = theap->profile_sample_rate; // todo: adjust difference? + } + _mi_theap_update_sample_rate(theap); + return old_sample_rate; +} + +static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) { + mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler) : NULL); // don't overwrite unless it is NULL + return mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t, &heap->profiler, &previous, profiler); +} + +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { + mi_profiler_stop(profiler); + return mi_heap_set_profiler(heap,profiler); +} + +mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) { + static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; + mi_heap_set_profiler(heap,NULL); // first clear any previous one + mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one +} + +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + mi_profiler_stop(profiler); + mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler) : NULL); // don't overwrite unless it is NULL + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { + mi_heap_set_profiler(heap,profiler); + } + } + return true; +} + +mi_decl_export bool mi_profile( mi_profiler_t* profiler) { + return mi_subproc_profile(mi_subproc_main(),profiler); +} + +bool mi_profiler_start(mi_profiler_t* profiler ) { + if (profiler==NULL) return false; + const bool was_running = mi_profiler_set_enabled(profiler,true); + if (was_running) return true; + // for the main heap, if this is the profiler, start the theap more aggressively + // otherwise it will be picked up when theaps take the slow generic malloc path. + mi_heap_t* heap = mi_heap_main(); + if (mi_heap_profiler(heap)==profiler) { + mi_theap_t* theap = _mi_heap_theap_peek(heap); + if (theap!=NULL) { + _mi_theap_set_profile_sample_rate(theap,1); + } + } + return false; +} + +bool mi_profiler_stop(mi_profiler_t* profiler) { + if (profiler==NULL) return true; + return mi_profiler_set_enabled(profiler,false); +} diff --git a/src/static.c b/src/static.c index 3554a4bc6..8a4f8c3d5 100644 --- a/src/static.c +++ b/src/static.c @@ -30,15 +30,17 @@ terms of the MIT license. A copy of the license can be found in the file #include "libc.c" #include "options.c" #include "os.c" +#include "prim/prim.c" // includes platform specific prim//prim.c +#include "prim/prim-tls.c" +#if MI_OSX_ZONE +#include "prim/osx/alloc-override-zone.c" +#endif #include "page.c" // includes page-queue.c #include "page-map.c" #include "random.c" +#include "sample-guarded.c" +#include "sample-profile.c" #include "stats.c" #include "subproc.c" #include "theap.c" #include "threadlocal.c" -#include "prim/prim.c" -#include "prim/prim-tls.c" -#if MI_OSX_ZONE -#include "prim/osx/alloc-override-zone.c" -#endif diff --git a/src/stats.c b/src/stats.c index 9220bfec2..c763bfc3c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -39,26 +39,21 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { if (amount > 0) { stat->total += amount; } } - -void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) { - mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); -} - -void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { - stat->total += amount; -} - -void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update_mt(stat, (int64_t)amount); } -void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update(stat, (int64_t)amount); } -void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update_mt(stat, -((int64_t)amount)); } -void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update(stat, -((int64_t)amount)); } @@ -69,9 +64,9 @@ void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // adjust atomically - const size_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); + const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); mi_atomic_addi64_relaxed(&stat->current, amount); - const size_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); + const int64_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } } @@ -82,16 +77,20 @@ static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { stat->total += amount; } -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust_mt(stat, (int64_t)amount); } -void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust(stat, (int64_t)amount); } -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust_mt(stat, -((int64_t)amount)); } -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust(stat, -((int64_t)amount)); } @@ -128,7 +127,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // copy all fields MI_STAT_FIELDS() - #if MI_STAT>1 + #if MI_STATS for (size_t i = 0; i <= MI_BIN_HUGE; i++) { mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]); } @@ -195,28 +194,27 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar } static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { - _mi_fprintf(out, arg," %-10s:", msg); + _mi_fprintf(out, arg," %-12s:", msg); if (unit != 0) { - if (unit > 0) { + if (unit > 0) { // as KiB (power of two) mi_print_amount(stat->peak, unit, out, arg); mi_print_amount(stat->total, unit, out, arg); - // mi_print_amount(stat->freed, unit, out, arg); mi_print_amount(stat->current, unit, out, arg); mi_print_amount(unit, 1, out, arg); mi_print_count(stat->total, unit, out, arg); } - else { + else if (unit==-1) { // as K (decimal) mi_print_amount(stat->peak, -1, out, arg); mi_print_amount(stat->total, -1, out, arg); - // mi_print_amount(stat->freed, -1, out, arg); mi_print_amount(stat->current, -1, out, arg); - if (unit == -1) { - _mi_fprintf(out, arg, "%24s", ""); - } - else { - mi_print_amount(-unit, 1, out, arg); - mi_print_count((stat->total / -unit), 0, out, arg); - } + _mi_fprintf(out, arg, "%24s", ""); + } + else { // as KiB , unit is final count + mi_print_amount(stat->peak, 1, out, arg); + mi_print_amount(stat->total, 1, out, arg); + mi_print_amount(stat->current, -1, out, arg); + _mi_fprintf(out, arg, "%12s", ""); + mi_print_count(-unit, 0, out, arg); } if (stat->current != 0) { _mi_fprintf(out, arg, " "); @@ -239,23 +237,23 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_stat_print_ex(stat, msg, unit, out, arg, NULL); } -#if MI_STAT>1 -static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { - _mi_fprintf(out, arg, " %-10s:", msg); +#if MI_STATS +static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, " %-12s:", msg); _mi_fprintf(out, arg, "%12s", " "); // no peak - mi_print_amount(stat->total, unit, out, arg); + mi_print_amount(stat->total, 1, out, arg); _mi_fprintf(out, arg, "\n"); } #endif static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); mi_print_amount(stat->total, 0, out, arg); _mi_fprintf(out, arg, "\n"); } static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); mi_print_amount(stat->total, 1, out, arg); _mi_fprintf(out, arg, "\n"); } @@ -264,16 +262,16 @@ static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count)); const int64_t avg_whole = avg_tens/10; const int64_t avg_frac1 = avg_tens%10; - _mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); + _mi_fprintf(out, arg, " %-12s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); } static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n", + _mi_fprintf(out, arg, " %-13s %11s %11s %11s %11s %11s\n", name, "peak ", "total ", "current ", "block ", "total# "); } -#if MI_STAT>1 +#if MI_STATS static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) { bool found = false; char buf[64]; @@ -342,8 +340,8 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_ size_t peak_commit; size_t page_faults; mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); - _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); - _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", + _mi_fprintf(out, arg, " %-12s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); + _mi_fprintf(out, arg, " %-12s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); if (peak_commit > 0) { @@ -365,19 +363,21 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou _mi_fprintf(out, arg, "%s %zu\n", name, id); if (stats->malloc_normal.total + stats->malloc_huge.total != 0) { - #if MI_STAT>1 + #if MI_STATS mi_print_header("blocks", out, arg); mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); #endif - #if MI_STAT - mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg); - mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg); + #if MI_STATS + mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg); + mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg); mi_stat_count_t total = { 0,0,0 }; mi_stat_count_add_mt(&total, &stats->malloc_normal); mi_stat_count_add_mt(&total, &stats->malloc_huge); - mi_stat_print_ex(&total, "total", 1, out, arg, ""); - #if MI_STAT>1 - mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); + mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); + #if MI_STATS>=2 + mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg); + #else + mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg); #endif _mi_fprintf(out, arg, "\n"); #endif @@ -391,13 +391,18 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); mi_stat_print(&stats->pages, "pages", 0, out, arg); mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg); + mi_stat_print(&stats->pages_os_abandoned, "os abandoned", 0, out, arg); + mi_stat_print(&stats->pages_os_allocated, "os allocated", 0, out, arg); mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg); mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg); mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg); mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg); mi_stat_counter_print(&stats->pages_extended, "extended", out, arg); mi_stat_counter_print(&stats->pages_retire, "retire", out, arg); + mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg); + mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); + mi_stat_counter_print(&stats->profile_samples, "prof samples", out, arg); _mi_fprintf(out, arg, "\n"); } @@ -423,7 +428,7 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_print_header("process", out, arg); mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); - _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + _mi_fprintf(out, arg, " %-12s: %5i\n", "numa nodes", _mi_os_numa_node_count()); mi_process_info_print_out(out, arg); _mi_fprintf(out, arg, "\n"); @@ -631,6 +636,11 @@ bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept { return mi_stats_copy(stats, mi_theap_get_stats(theap)); } +void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept { + if (theap == NULL) return; + _mi_theap_merge_stats(theap); +} + static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) { mi_stats_t* stats = (mi_stats_t*)arg; mi_stats_add_into(stats, mi_heap_get_stats(heap)); diff --git a/src/subproc.c b/src/subproc.c index 7cce9edd7..f42d4a11c 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -102,7 +102,7 @@ bool _mi_subproc_is_main(mi_subproc_t* subproc) { mi_subproc_t* _mi_subproc(void) { mi_theap_t* theap = _mi_theap_default(); - if (theap == NULL || theap->tld == NULL) { // see issue #1289 + if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 and #1391 return _mi_subproc_main(); } else { @@ -228,8 +228,8 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro } } - subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap - + subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + if (!_mi_subproc_is_main(subproc)) { // merge stats back into the main subproc _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); diff --git a/src/theap.c b/src/theap.c index d9fb4309d..1731830a7 100644 --- a/src/theap.c +++ b/src/theap.c @@ -99,7 +99,8 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag MI_UNUSED(theap); mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); - _mi_page_free_collect(page, collect >= MI_FORCE); + _mi_page_free_collect(page, collect >= MI_FORCE); // update used count + _mi_page_update_stats(page); if (mi_page_all_free(page)) { // no more used blocks, possibly free the page. if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired @@ -115,7 +116,7 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag } void _mi_theap_merge_stats(mi_theap_t* theap) { - mi_assert_internal(mi_theap_is_initialized(theap)); + mi_assert_internal(mi_theap_is_initialized(theap)); mi_heap_t* const heap = _mi_theap_heap(theap); _mi_stats_merge_into(&heap->stats, &theap->stats); } @@ -144,7 +145,9 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) } // merge statistics - _mi_theap_merge_stats(theap); + if (mi_option_is_enabled(mi_option_collect_merges_stats)) { + _mi_theap_merge_stats(theap); + } } void _mi_theap_collect_abandon(mi_theap_t* theap) { @@ -187,43 +190,6 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { return previous; } -#if MI_GUARDED -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = sample_rate; - theap->guarded_sample_count = sample_rate; // count down samples - if (theap->guarded_sample_rate > 1) { - if (seed == 0) { - seed = _mi_theap_random_next(theap); - } - theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` - } -} - -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - theap->guarded_size_min = min; - theap->guarded_size_max = (min > max ? min : max); -} - -static void mi_theap_guarded_init(mi_theap_t* theap) { - mi_theap_guarded_set_sample_rate(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), - (size_t)mi_option_get(mi_option_guarded_sample_seed)); - mi_theap_guarded_set_size_bound(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), - (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); -} -#else -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); -} - -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); -} -static void mi_theap_guarded_init(mi_theap_t* theap) { - MI_UNUSED(theap); -} -#endif static void mi_theap_options_init(mi_theap_t* theap) { theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); @@ -287,7 +253,7 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random } // theap->cookie = _mi_theap_random_next(theap) | 1; - mi_theap_guarded_init(theap); // needs theap->random + _mi_theap_guarded_init(theap); // needs theap->random if (!theap->is_detached) { mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem } @@ -545,7 +511,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { area->reserved = page->reserved * bsize; area->committed = page->capacity * bsize; area->blocks = mi_page_start(page); - area->used = page->used; // number of blocks in use (#553) + area->used = mi_page_used(page); // number of blocks in use (#553) area->block_size = ubsize; area->full_block_size = bsize; area->reserved1 = page; @@ -571,7 +537,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi _mi_page_free_collect(page,true); // collect both thread_delayed and local_free mi_assert_internal(page->local_free == NULL); - if (page->used == 0) return true; + if (mi_page_used(page) == 0) return true; size_t psize; uint8_t* const pstart = mi_page_area(page, &psize); @@ -581,13 +547,13 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi // optimize page with one block if (page->capacity == 1) { - mi_assert_internal(page->used == 1 && page->free == NULL); + mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL); return visitor(heap, area, pstart, ubsize, arg); } mi_assert(bsize <= UINT32_MAX); // optimize full pages - if (page->used == page->capacity) { + if (mi_page_used(page) == page->capacity) { uint8_t* block = pstart; for (size_t i = 0; i < page->capacity; i++) { if (!visitor(heap, area, block, ubsize, arg)) return false; @@ -631,7 +597,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); free_map[bitidx] |= ((uintptr_t)1 << bit); } - mi_assert_internal(page->capacity == (free_count + page->used)); + mi_assert_internal(page->capacity == (free_count + mi_page_used(page))); // walk through all blocks skipping the free ones #if MI_DEBUG>1 @@ -663,7 +629,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi block += bsize * MI_INTPTR_BITS; } } - mi_assert_internal(page->used == used_count); + mi_assert_internal(mi_page_used(page) == used_count); return true; } diff --git a/src/threadlocal.c b/src/threadlocal.c index 355498cee..6f280d093 100644 --- a/src/threadlocal.c +++ b/src/threadlocal.c @@ -174,7 +174,6 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { // get a tls slot value static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) { - mi_assert_internal(key!=0); const mi_thread_locals_t* const tls = mi_thread_locals_peek(); if mi_unlikely(tls==NULL) { // this can happen if a thread local is accessed after the thread local has been freed @@ -193,12 +192,13 @@ static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key // get a thread local value void* _mi_thread_local_get( mi_thread_local_t key ) { - mi_assert_internal(key!=0); if mi_likely(key == mi_thread_local_key_fast) { return mi_slot_fast_get(); } else { - return mi_thread_local_get_regular(key); + void* p = mi_thread_local_get_regular(key); + if (key==0) { mi_assert_internal(p==NULL); } + return p; } } diff --git a/test/main-override-static.c b/test/main-override-static.c index c3230e1af..368710c3c 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -50,11 +50,11 @@ int main() { // double_free1(); // double_free2(); // corrupt_free(); + // invalid_free(); // block_overflow1(); // block_overflow2(); test_canary_leak(); // test_aslr(); - // invalid_free(); // test_reserved(); // negative_stat(); // test_theap_walk(); @@ -349,7 +349,7 @@ static void test_large_pages(void) { #include #include -#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_INTPTR_SIZE) +#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_SIZE_SIZE) #define MI_BIN_HUGE 100 //#define MI_ALIGN2W @@ -390,10 +390,10 @@ static inline uint8_t mi_bsr32(uint32_t x) { // Bit scan reverse: return the index of the highest bit. uint8_t _mi_bsr(uintptr_t x) { if (x == 0) return 0; - #if MI_INTPTR_SIZE==8 + #if MI_SIZE_SIZE==8 uint32_t hi = (x >> 32); return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); - #elif MI_INTPTR_SIZE==4 + #elif MI_SIZE_SIZE==4 return mi_bsr32(x); #else # error "define bsr for non-32 or 64-bit platforms" diff --git a/test/test-api.c b/test/test-api.c index 09a663c33..70bba60b1 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -115,7 +115,7 @@ int main(void) { #endif CHECK_BODY("calloc-overflow") { // use (size_t)&mi_calloc to get some number without triggering compiler warnings - result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); + result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); }; CHECK_BODY("malloc-large") { // see PR #544. void* p = mi_malloc(67108872); @@ -132,7 +132,7 @@ int main(void) { CHECK_BODY("mi_urealloc_invalid") { void* p = mi_malloc(64); size_t pre, post; - void* q = mi_urealloc((char*)p + 3, 32, &pre, &post); + void* q = mi_urealloc((char*)p + 3, 40, &pre, &post); mi_free(p); result = (q==NULL || q==(uint8_t*)p+3); } @@ -377,6 +377,7 @@ int main(void) { ptr = mi_rezalloc_aligned(ptr, n, alignment); assert(mem_has_vals((uint8_t*)ptr,n/2,123)); result = mem_is_zero((uint8_t*)ptr + n/2, n/2); + mi_free(ptr); } // --------------------------------------------------- @@ -414,6 +415,7 @@ int main(void) { shared_ptr = mi_realloc(shared_ptr, i * 64); } } + mi_free(shared_ptr); } // --------------------------------------------------- @@ -474,7 +476,7 @@ int main(void) { } } - #if (MI_INTPTR_SIZE > 4) + #if (MI_SIZE_SIZE > 4) CHECK_BODY("arena_reserve") { result = (0==mi_reserve_os_memory(16*MI_GiB,false,true)); } diff --git a/test/test-profile.c b/test/test-profile.c new file mode 100644 index 000000000..41c636622 --- /dev/null +++ b/test/test-profile.c @@ -0,0 +1,189 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026-2026, Microsoft Research, Daniel Schwartz-Narbonne, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// Tests for the mimalloc heap profiler (src/profile.c). + +#include +#include +#include +#include +#include + +#include "mimalloc.h" +#include "mimalloc-profile.h" +#include "mimalloc/internal.h" +#include "testhelper.h" + +// --------------------------------------------------------------------------- +// Shared callback state (not thread safe!) +// --------------------------------------------------------------------------- + +typedef struct { + mi_profiler_t profiler; + uint64_t alloc_count; + uint64_t free_count; + size_t last_size; + uint64_t last_upscaled; + void* last_ptr; +} my_profiler_t; + +static inline my_profiler_t* downcast( mi_profiler_t* prof ) { + return (my_profiler_t*)prof; +} + +// We store ptr in user_data so on_free can verify the round-trip. + +#define TEST_THRESHOLD (16 * 1024) + +static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap) { + MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(requested_size); + my_profiler_t* prof = downcast(profiler); + assert(bytes_since_last_sample >= requested_size); + prof->alloc_count++; + prof->last_ptr = ptr; + prof->last_size = requested_size; + prof->last_upscaled = bytes_since_last_sample; + // store ptr to verify round-trip + assert(data->user_data_size >= sizeof(void*)); + assert(data->user_data_size >= prof->profiler.sample_data_size); + data->user_data[0] = ptr; + return TEST_THRESHOLD; +} + +static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, const mi_heap_t* heap) { + MI_UNUSED(heap); MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); + my_profiler_t* prof = downcast(profiler); + prof->free_count++; + // verify the user_data round-trip + assert(data->user_data[0] == ptr); +} + +static my_profiler_t my_profiler = { + { // profiler_t + NULL, // reserved + sizeof(void*), // needed sample data size + 0, // initial sample rate (default) + &on_alloc, + &on_free, + NULL + }, + 0, 0, 0, 0, NULL +}; + + + + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Force at least one sample by allocating well over the threshold. +static void allocate_past_threshold(void) { + size_t total = 0; + while (total < TEST_THRESHOLD * 10) { + void* p = mi_malloc(4096); + mi_free(p); + total += 4096; + // mi_collect() + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +bool test_profiler_samples(void) { + CHECK_BODY("profiler: on_alloc called after threshold") { + uint64_t before = my_profiler.alloc_count; + allocate_past_threshold(); + result = (my_profiler.alloc_count > before); + } + return true; +} + +#define MAXLOOP 100000 + +bool test_profiler_record_fields(void) { + CHECK_BODY("profiler: record ptr and size are non-zero") { + uint64_t before = my_profiler.alloc_count; + int count; + for (count = 0; my_profiler.alloc_count == before && count < MAXLOOP; count++) { + void* p = mi_malloc(1024); + mi_free(p); + } + assert(count!=MAXLOOP); + result = (my_profiler.last_ptr != NULL && my_profiler.last_size > 0 && my_profiler.last_upscaled > 0 && count!=MAXLOOP); + } + return true; +} + +bool test_profiler_on_free_called(void) { + CHECK_BODY("profiler: on_free called for sampled allocation") { + uint64_t alloc_before = my_profiler.alloc_count; + uint64_t free_before = my_profiler.free_count; + + // Keep the pointer live until we confirm a sample was taken, then free it. + void* sampled = NULL; + int count; + for (count = 0; my_profiler.alloc_count == alloc_before && count < MAXLOOP; count++) { + if (sampled) { mi_free(sampled); } + sampled = mi_malloc(1024); + } + // At this point my_profiler.last_ptr is the sampled pointer. + // Free it and check on_free fires. + void* expected = my_profiler.last_ptr; + mi_free(expected); + sampled = NULL; + assert(count!=MAXLOOP); + result = (my_profiler.free_count > free_before && count!=MAXLOOP); + } + return true; +} + +bool test_profiler_upscaled_at_least_size(void) { + CHECK_BODY("profiler: upscaled_size >= size") { + uint64_t before = my_profiler.alloc_count; + int count; + for (count = 0; my_profiler.alloc_count == before && count < MAXLOOP; count++) { + void* p = mi_malloc(256); + mi_free(p); + } + assert(count!=MAXLOOP); + result = (my_profiler.last_upscaled >= my_profiler.last_size && count!=MAXLOOP); + } + return true; +} + +bool test_profiler_free_count_le_alloc_count(void) { + CHECK_BODY("profiler: on_free never called more times than on_alloc") { + // Free can only fire for sampled allocations, so free_count <= alloc_count + // must hold at all times. + allocate_past_threshold(); + result = (my_profiler.free_count <= my_profiler.alloc_count); + } + return true; +} + + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +int main(void) { + mi_profile(&my_profiler.profiler); + mi_profiler_start(&my_profiler.profiler); + + test_profiler_upscaled_at_least_size(); + test_profiler_samples(); + test_profiler_record_fields(); + test_profiler_on_free_called(); + test_profiler_free_count_le_alloc_count(); + + mi_profiler_stop(&my_profiler.profiler); + + return print_test_summary(); +}