From cdde2ec49e00b6f950f8150e5ff948b4a2a0d63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 18 Sep 2026 20:47:22 -0300 Subject: [PATCH 1/2] Fix file associations in macOS. --- premake4.lua | 7 ++++- premake5.lua | 6 ++++- src/eepp/system/fileassociation.cpp | 28 ++++++++++++++------ src/eepp/system/fileassociation_macos.mm | 33 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 src/eepp/system/fileassociation_macos.mm diff --git a/premake4.lua b/premake4.lua index 3b74bcbb4..e73acffec 100644 --- a/premake4.lua +++ b/premake4.lua @@ -593,6 +593,10 @@ function build_link_configuration( package_name, use_ee_icon ) end end + if os.is_real("macosx") then + linkoptions { "-weak_framework UniformTypeIdentifiers" } + end + if _OPTIONS["with-mold-linker"] then if _OPTIONS.platform == "clang" or _OPTIONS.platform == "clang-analyzer" then linkoptions { "-fuse-ld=mold" } @@ -1644,7 +1648,8 @@ solution "eepp" set_targetdir("libs/" .. os.get_real() .. "/") includedirs { "include", "src" } files { "src/eepp/ui/platform/macos/macosmenubar.mm", - "src/eepp/window/platform/macos/platformhelper.mm" } + "src/eepp/window/platform/macos/platformhelper.mm", + "src/eepp/system/fileassociation_macos.mm" } buildoptions { "-x objective-c++" } if not is_vs() then buildoptions{ "-std=c++20" } diff --git a/premake5.lua b/premake5.lua index a83ffea23..28a88c687 100644 --- a/premake5.lua +++ b/premake5.lua @@ -538,6 +538,9 @@ function build_link_configuration( package_name, use_ee_icon ) linkoptions { "-Wl,-rpath,'$$ORIGIN'" } end + filter "system:macosx" + linkoptions { "-weak_framework UniformTypeIdentifiers" } + filter { "system:bsd" } if package_name ~= "eepp" and package_name ~= "eepp-static" then if type(userelativelinks) == "function" then @@ -1675,7 +1678,8 @@ workspace "eepp" cppdialect "C++20" incdirs { "include", "src" } files { "src/eepp/ui/platform/macos/macosmenubar.mm", - "src/eepp/window/platform/macos/platformhelper.mm" } + "src/eepp/window/platform/macos/platformhelper.mm", + "src/eepp/system/fileassociation_macos.mm" } buildoptions { "-x objective-c++" } build_base_cpp_configuration( "eepp-macos-helper" ) target_dir_lib( "" ) diff --git a/src/eepp/system/fileassociation.cpp b/src/eepp/system/fileassociation.cpp index eb0426606..b0b6c4d9b 100644 --- a/src/eepp/system/fileassociation.cpp +++ b/src/eepp/system/fileassociation.cpp @@ -21,6 +21,10 @@ #include #endif +#if EE_PLATFORM == EE_PLATFORM_MACOS +extern "C" CFStringRef eeppFileAssociationTypeIdentifierForExtension( const char* extension ); +#endif + namespace EE::System { namespace { @@ -557,11 +561,19 @@ static CFStringRef cfString( const std::string& value ) { } static CFStringRef typeForExtension( const std::string& extension ) { - CFRef extensionString( cfString( extension ) ); - if ( !extensionString.get() ) - return nullptr; - return UTTypeCreatePreferredIdentifierForTag( - kUTTagClassFilenameExtension, static_cast( extensionString.get() ), nullptr ); + return eeppFileAssociationTypeIdentifierForExtension( extension.c_str() ); +} + +static CFStringRef applicationIdentifier( const FileAssociationApplication& application ) { + /* Launch Services uses the bundle identifier from the registered bundle. The cross-platform + * application id is not necessarily that identifier (ecode, for example, uses ensoft.dev in + * its macOS Info.plist). Prefer the bundle metadata when this process is running from a bundle, + * and retain the supplied id as a fallback for non-bundled callers. */ + if ( auto* bundle = CFBundleGetMainBundle() ) { + if ( auto identifier = CFBundleGetIdentifier( bundle ) ) + return static_cast( CFRetain( identifier ) ); + } + return cfString( application.id ); } #endif @@ -621,7 +633,7 @@ std::vector FileAssociation::getRegisteredExtensions( registered.emplace_back( extension ); } #elif EE_PLATFORM == EE_PLATFORM_MACOS - CFRef applicationId( cfString( mApplication.id ) ); + CFRef applicationId( applicationIdentifier( mApplication ) ); if ( !applicationId.get() ) { mLastError = "The application identifier is not valid UTF-8."; return {}; @@ -656,7 +668,7 @@ bool FileAssociation::setRegisteredExtensions( const std::vector& r const auto supported = normalizeExtensions( supportedExtensions ); const auto requested = normalizeExtensions( registeredExtensions ); std::vector selected; - selected.reserve( (std::min)( requested.size(), supported.size() ) ); + selected.reserve( ( std::min )( requested.size(), supported.size() ) ); std::set_intersection( requested.begin(), requested.end(), supported.begin(), supported.end(), std::back_inserter( selected ) ); #if EE_PLATFORM == EE_PLATFORM_LINUX || EE_PLATFORM == EE_PLATFORM_BSD @@ -712,7 +724,7 @@ bool FileAssociation::setRegisteredExtensions( const std::vector& r mLastError = "Launch Services could not register the application bundle."; return false; } - CFRef applicationId( cfString( mApplication.id ) ); + CFRef applicationId( applicationIdentifier( mApplication ) ); if ( !applicationId.get() ) { mLastError = "The application identifier is not valid UTF-8."; return false; diff --git a/src/eepp/system/fileassociation_macos.mm b/src/eepp/system/fileassociation_macos.mm new file mode 100644 index 000000000..dd449ae15 --- /dev/null +++ b/src/eepp/system/fileassociation_macos.mm @@ -0,0 +1,33 @@ +#include + +#if EE_PLATFORM == EE_PLATFORM_MACOS + +#import +#import +#import + +extern "C" CFStringRef eeppFileAssociationTypeIdentifierForExtension( const char* extension ) { + if ( extension == nullptr ) + return nullptr; + + NSString* extensionString = [NSString stringWithUTF8String:extension]; + if ( extensionString == nil ) + return nullptr; + + if ( @available( macOS 11.0, * ) ) { + UTType* type = [UTType typeWithFilenameExtension:extensionString]; + if ( type == nil || type.identifier == nil ) + return nullptr; + return CFStringCreateWithCString( kCFAllocatorDefault, type.identifier.UTF8String, + kCFStringEncodingUTF8 ); + } + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + CFStringRef type = UTTypeCreatePreferredIdentifierForTag( + kUTTagClassFilenameExtension, static_cast( extensionString ), nullptr ); +#pragma clang diagnostic pop + return type; +} + +#endif From d885c6b31a0708b38c1da9742c2ae1326fbe39f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sat, 19 Sep 2026 21:52:07 -0300 Subject: [PATCH 2/2] Update svector and unordered_dense to its latest releases. --- include/eepp/thirdparty/huge_page_allocator.h | 260 ++ include/eepp/thirdparty/stl.h | 6 +- include/eepp/thirdparty/svector.h | 1524 +++++++- include/eepp/thirdparty/unordered_dense.h | 3238 ++++++++++++++--- 4 files changed, 4299 insertions(+), 729 deletions(-) create mode 100644 include/eepp/thirdparty/huge_page_allocator.h diff --git a/include/eepp/thirdparty/huge_page_allocator.h b/include/eepp/thirdparty/huge_page_allocator.h new file mode 100644 index 000000000..676c84c4a --- /dev/null +++ b/include/eepp/thirdparty/huge_page_allocator.h @@ -0,0 +1,260 @@ +///////////////////////// ankerl::unordered_dense::huge_page_allocator ///////////////////////// + +// An opt-in allocator that puts large blocks on transparent huge pages. +// Version 5.0.1 +// https://github.com/martinus/unordered_dense +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2022 Martin Leitner-Ankerl +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ANKERL_UNORDERED_DENSE_HUGE_PAGE_ALLOCATOR_H +#define ANKERL_UNORDERED_DENSE_HUGE_PAGE_ALLOCATOR_H + +// A separate header rather than a section of unordered_dense.h, because it needs . +#include "unordered_dense.h" // for the version namespace and ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS + +#include // for size_t, ptrdiff_t +#include // for uintptr_t +#include // for abort +#include // for equal_to +#include // for allocator +#include // for bad_alloc +#include // for true_type +#include // for pair + +#if defined(__linux__) && defined(__has_include) +# if __has_include() +# include // for mmap, munmap, madvise, MADV_HUGEPAGE +# define ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE 1 // NOLINT(cppcoreguidelines-macro-usage) +# endif +#endif +#if !defined(ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE) +# define ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE 0 // NOLINT(cppcoreguidelines-macro-usage) +#endif + +namespace ankerl::unordered_dense { +inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE { + +// What it is for. This map touches two or three regions per operation -- the group block, the +// value it points at, and a string's body -- and every one of them is a random address, so on 4 KB +// pages every one is an address translation. Zen 4's first-level data TLB holds 72 entries: 288 KB. +// Any table larger than that pays an L2 TLB lookup per access, and on a dependent chain -- probe, +// then value, then an erase's second walk -- that lookup is latency. Measured on the scored +// benchmark by running the same binary with the heap on 2 MB pages: 2.6% under gcc and 3.6% under +// clang over all fifteen workloads, 5-8% on churn and on the 50% find at 50000 entries, and 22% of +// a lookup past the last-level cache (notes/index-design.md, "The score runs on 4 KB pages"). +// +// What it does. An allocation of at least `Threshold` bytes is `mmap`ed on its own, aligned to 2 MB +// and rounded up to a multiple of it, and `madvise(MADV_HUGEPAGE)`d, which is what asks the kernel +// for huge pages when `/sys/kernel/mm/transparent_hugepage/enabled` is `madvise` -- the default on +// most distributions, and the mode under which a map on `std::allocator` never sees one. Smaller +// allocations go to `std::allocator`. Hand it to the map as the allocator and both regions get +// it, since the index rebinds the value allocator: +// +// ankerl::unordered_dense::map, std::equal_to, +// ankerl::unordered_dense::huge_page_allocator>> +// +// What it cannot do, and it is the reason the threshold cannot go below 2 MB. Huge pages are 2 MB +// each, whole: a 360 KB index cannot be on one by itself. The score's 50000-entry tables got theirs +// from glibc, which `madvise`s the *heap* so that neighbouring small blocks share an extent +// (`GLIBC_TUNABLES=glibc.malloc.hugetlb=1`, glibc 2.35 and later) -- an allocator that owns only its +// own blocks has no neighbour to share with. So this pays off once the blocks themselves are 2 MB: +// an index of about 370000 entries and up, a `std::vector` of values from 2 MB / sizeof(value_type) +// entries and up. Below that, the environment route is the one that works, and the "Huge Pages" +// section of https://github.com/martinus/unordered_dense/blob/main/doc/usage.md says so. +// +// What it costs. Every block is rounded up to 2 MB, and under `MADV_HUGEPAGE` touching one byte of +// an aligned 2 MB extent populates all of it, so the rounding is resident memory, not just address +// space: a 2.1 MB block occupies 4 MB. The map doubles both of its regions, so at most half of one +// doubling step is lost per region, and only while that region is between doublings. It also means +// a block below the threshold is never rounded, which is what the threshold is for. +// +// Where it is a plain std::allocator. Anywhere without and MADV_HUGEPAGE, which is +// Windows and macOS: the class exists with the same interface, `uses_huge_pages` is false, and +// everything is forwarded, so code written against it compiles everywhere and is only faster where +// the kernel can help. Windows has VirtualAlloc with MEM_LARGE_PAGES, which needs the +// SeLockMemoryPrivilege that an ordinary process does not have, and macOS has superpages through +// its own mmap flags; neither is asked for here. +// +// The allocator is stateless, so instances compare equal, a container copy takes its own, and +// propagation is never a question. The threshold is a template parameter for that reason -- a +// runtime member would make instances differ and make every propagation trait matter. +template +class huge_page_allocator { +public: + static constexpr std::size_t huge_page_size = std::size_t{2} << 20U; + static constexpr std::size_t threshold = Threshold; + static constexpr bool uses_huge_pages = ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE != 0; + static_assert(Threshold >= huge_page_size, "a block below one huge page cannot be on a huge page of its own"); + + using value_type = T; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using propagate_on_container_move_assignment = std::true_type; + using is_always_equal = std::true_type; + + template + struct rebind { + using other = huge_page_allocator; + }; + + constexpr huge_page_allocator() noexcept = default; + + template + // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions) + constexpr huge_page_allocator(huge_page_allocator const& /*other*/) noexcept {} + + [[nodiscard]] auto allocate(std::size_t n) -> T* { + if (!is_huge(n)) { + return std::allocator{}.allocate(n); + } +#if ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE + return static_cast(map_huge(rounded_bytes(n))); +#else + return std::allocator{}.allocate(n); +#endif + } + + void deallocate(T* p, std::size_t n) noexcept { + if (!is_huge(n)) { + std::allocator{}.deallocate(p, n); + return; + } +#if ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE + ::munmap(static_cast(p), rounded_bytes(n)); +#else + std::allocator{}.deallocate(p, n); +#endif + } + + template + [[nodiscard]] friend constexpr auto operator==(huge_page_allocator const& /*a*/, + huge_page_allocator const& /*b*/) noexcept -> bool { + return Threshold == Th; + } + template + [[nodiscard]] friend constexpr auto operator!=(huge_page_allocator const& a, huge_page_allocator const& b) noexcept + -> bool { + return !(a == b); + } + + // Whether a request for n objects takes the huge page path. Deallocate has to make exactly the + // same decision from the same n, which is why it is a pure function of n and the types. + [[nodiscard]] static constexpr auto is_huge(std::size_t n) noexcept -> bool { + return uses_huge_pages && n >= Threshold / sizeof(T) && n * sizeof(T) >= Threshold; + } + +private: + [[nodiscard]] static constexpr auto rounded_bytes(std::size_t n) noexcept -> std::size_t { + auto const bytes = n * sizeof(T); + return (bytes + huge_page_size - 1) / huge_page_size * huge_page_size; + } + +#if ANKERL_UNORDERED_DENSE_HAS_MADV_HUGEPAGE + [[noreturn]] static void out_of_memory() { +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() + throw std::bad_alloc(); +# else + std::abort(); +# endif + } + + // mmap gives page alignment, not 2 MB alignment, so this maps one huge page too many, unmaps + // the misaligned head and the tail it no longer needs, and is left holding exactly + // [aligned, aligned + len). deallocate then unmaps that range and nothing else needs to be + // remembered. The madvise is advice: if the kernel cannot or will not, the block is ordinary + // 4 KB pages and still correct, which is why its result is not checked. + [[nodiscard]] static auto map_huge(std::size_t len) -> void* { + auto const span = len + huge_page_size; + auto* raw = ::mmap(nullptr, span, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) { // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,performance-no-int-to-ptr) + out_of_memory(); + } + auto const start = reinterpret_cast(raw); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + auto const aligned = (start + huge_page_size - 1) / huge_page_size * huge_page_size; + auto const head = aligned - start; + auto const tail = span - head - len; + if (head != 0) { + ::munmap(raw, head); + } + if (tail != 0) { + ::munmap(reinterpret_cast(aligned + len), + tail); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) + } + auto* block = + reinterpret_cast(aligned); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) + ::madvise(block, len, MADV_HUGEPAGE); + return block; + } +#endif +}; + +// The four containers with this allocator already in the allocator slot, so that opting in is one +// word rather than five template arguments -- the shape `pmr::` has in unordered_dense.h. The +// threshold stays at its default here; a different one is still the allocator written out. +namespace huge_page { + +template , class KeyEqual = std::equal_to, class Bucket = bucket_type::group> +using map = detail::table>, Bucket, false>; + +// The segment size is worth setting here, and is not defaulted to a large one: a segment is +// allocated whole, so a 16 MB default would cost a map with ten elements 16 MB. The default gets +// this allocator for the index and not for the values, which is the right floor for a small map and +// the wrong answer for a large one -- 16 MB is at least one whole huge page after the rounding for +// any element size, and is what doc/usage.md recommends in the repository. +template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::group, + std::size_t MaxSegmentSizeBytes = default_segment_size_bytes> +using segmented_map = detail::table< + Key, + T, + Hash, + KeyEqual, + detail::segmented_container_for, huge_page_allocator>, MaxSegmentSizeBytes>, + Bucket, + true>; + +template , class KeyEqual = std::equal_to, class Bucket = bucket_type::group> +using set = detail::table, Bucket, false>; + +template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::group, + std::size_t MaxSegmentSizeBytes = default_segment_size_bytes> +using segmented_set = detail::table, MaxSegmentSizeBytes>, + Bucket, + true>; + +} // namespace huge_page + +} // namespace ANKERL_UNORDERED_DENSE_NAMESPACE +} // namespace ankerl::unordered_dense + +#endif diff --git a/include/eepp/thirdparty/stl.h b/include/eepp/thirdparty/stl.h index 264bcca88..96d14537c 100644 --- a/include/eepp/thirdparty/stl.h +++ b/include/eepp/thirdparty/stl.h @@ -1,7 +1,7 @@ ///////////////////////// ankerl::unordered_dense::{map, set} ///////////////////////// // A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion. -// Version 4.8.1 +// Version 5.0.1 // https://github.com/martinus/unordered_dense // // Licensed under the MIT License . @@ -77,7 +77,9 @@ #if defined(_MSC_VER) && defined(_M_X64) # include -# pragma intrinsic(_umul128) +# if !defined(_M_ARM64EC) +# pragma intrinsic(_umul128) +# endif #endif #endif diff --git a/include/eepp/thirdparty/svector.h b/include/eepp/thirdparty/svector.h index 926966083..7b8c0e10f 100644 --- a/include/eepp/thirdparty/svector.h +++ b/include/eepp/thirdparty/svector.h @@ -1,5 +1,5 @@ // ┌─┐┬ ┬┌─┐┌─┐┌┬┐┌─┐┬─┐ Compact SVO optimized vector C++17 or higher -// └─┐└┐┌┘├┤ │ │ │ │├┬┘ Version 1.0.3 +// └─┐└┐┌┘├┤ │ │ │ │├┬┘ Version 1.4.0 // └─┘ └┘ └─┘└─┘ ┴ └─┘┴└─ https://github.com/martinus/svector // // Licensed under the MIT License . @@ -29,8 +29,8 @@ // see https://semver.org/spec/v2.0.0.html #define ANKERL_SVECTOR_VERSION_MAJOR 1 // incompatible API changes -#define ANKERL_SVECTOR_VERSION_MINOR 0 // add functionality in a backwards compatible manner -#define ANKERL_SVECTOR_VERSION_PATCH 3 // backwards compatible bug fixes +#define ANKERL_SVECTOR_VERSION_MINOR 4 // add functionality in a backwards compatible manner +#define ANKERL_SVECTOR_VERSION_PATCH 0 // backwards compatible bug fixes // API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ #define ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch @@ -38,6 +38,21 @@ #define ANKERL_SVECTOR_NAMESPACE \ ANKERL_SVECTOR_VERSION_CONCAT(ANKERL_SVECTOR_VERSION_MAJOR, ANKERL_SVECTOR_VERSION_MINOR, ANKERL_SVECTOR_VERSION_PATCH) +/** + * @brief Keeps a function out of line even where a compiler would rather inline it. + * + * Only used for commit_grow(), where being shared between callers is the whole point, see + * insert_n(). Silently nothing on a compiler that has no such spelling: then the inlining is back + * and so is the code size, but nothing is wrong. + */ +#if defined(__GNUC__) || defined(__clang__) +# define ANKERL_SVECTOR_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +# define ANKERL_SVECTOR_NOINLINE __declspec(noinline) +#else +# define ANKERL_SVECTOR_NOINLINE +#endif + #include #include #include @@ -52,16 +67,48 @@ #include #include +#if defined(__has_include) +# if __has_include() +# include +# endif +#endif + +// The C++23 range members. Guarded on the library rather than on the language, because what they +// need is std::from_range_t and the range concepts, and a C++17 build has neither. Everything else +// in this header stays exactly as it was for such a build. +#if defined(__cpp_lib_containers_ranges) && __cpp_lib_containers_ranges >= 202202L +# define ANKERL_SVECTOR_HAS_RANGES 1 +# include +# include +#else +# define ANKERL_SVECTOR_HAS_RANGES 0 +#endif + namespace ankerl { inline namespace ANKERL_SVECTOR_NAMESPACE { namespace detail { template -using enable_if_t = typename std::enable_if::type; +using enable_if_t = std::enable_if_t; template using is_input_iterator = std::is_base_of::iterator_category>; +// A C++20 iterator need not publish an iterator_category, only an iterator_concept, and then +// is_input_iterator above cannot even be formed. Used to decide whether a range can be handed to +// the iterator pair members as it is. +template +struct has_iterator_category : std::false_type {}; + +template +struct has_iterator_category::iterator_category>> : std::true_type {}; + +#if ANKERL_SVECTOR_HAS_RANGES +// The standard calls this container-compatible-range and uses it for exactly these members. +template +concept container_compatible_range = std::ranges::input_range && std::convertible_to, T>; +#endif + constexpr auto round_up(size_t n, size_t multiple) -> size_t { return ((n + (multiple - 1)) / multiple) * multiple; } @@ -101,6 +148,277 @@ constexpr auto automatic_capacity(size_t min_inline_capacity) -> size_t { return cx_min((size_of_svector(min_inline_capacity) - 1U) / sizeof(T), size_t{127}); } +/** + * @brief Whether the allocator has anything to say about building and destroying an element. + * + * std::allocator_traits::construct() calls a.construct(p, args...) when that compiles and does a + * placement new otherwise, and destroy() is the same story with p->~T(). So when none of them + * compile, going through the traits and doing it directly are the same code, and a whole range can + * go through the algorithms in instead, which know how to turn the relocation of a + * trivially copyable T into a memcpy. std::allocator lands here, and so does the usual arena or + * pool allocator that only spells out allocate() and deallocate(), which is what keeps both of them + * on exactly the code there was before there was an allocator at all. + * std::pmr::polymorphic_allocator, which passes its resource on to the elements it builds, does + * not, and takes the loops below. + * + * See builds_directly() for which signatures are asked about and why the order matters. + */ +template +struct has_construct_impl : std::false_type {}; + +template +struct has_construct_impl().construct(std::declval()...))>, A, Args...> + : std::true_type {}; + +template +using has_construct = has_construct_impl; + +template +struct has_destroy : std::false_type {}; + +template +struct has_destroy().destroy(std::declval()))>> : std::true_type {}; + +/** + * @brief std::allocator, which has to be named rather than probed. + * + * Until C++20 it carries a construct() and a destroy() of its own, deprecated in C++17 and gone + * after it, and both do exactly the placement new and the ~T() the traits would have done without + * them. So it answers the probes above with a yes that means no, and letting them decide would take + * the default allocator down the generic path -- and with it every svector written before there was + * an allocator to name. + */ +template +struct is_std_allocator : std::false_type {}; + +template +struct is_std_allocator> : std::true_type {}; + +/** + * @brief The probes and the std::allocator exemption, put together so that none is asked that does + * not have to be. + * + * The probed signatures are the argument shapes the bulk operations use: nothing, an lvalue to copy, + * an rvalue to move. Inserting from a range of some other type builds a T from whatever the iterator + * yields, which is not probed, so an allocator with a construct() for that and for none of these + * three would be missed. std::allocator_traits asks the same question one expression at a time; this + * asks it once for all of them, because the answer picks a whole algorithm. + * + * std::disjunction and std::conjunction stop instantiating at the first argument that decides the + * answer, and that is load bearing rather than tidy: std::pmr::polymorphic_allocator::destroy() is + * deprecated in C++20, and merely naming it in an unevaluated expression is enough for a build with + * -Werror to fail. Its construct() is not deprecated and answers first, so destroy() is never asked + * about. std::allocator answers before any of them. + */ +template +struct builds_directly : std::disjunction, + std::conjunction>, + std::negation>, + std::negation>, + std::negation>>> {}; + +template +inline constexpr bool builds_elements_directly = builds_directly::value; + +/** + * @brief Whether the allocator's allocate() and deallocate() are ::operator new and ::operator + * delete, so storage below can call those and skip the trip through allocator_traits. + * + * Only std::allocator, and only because the standard says that is what it does. Deliberately not + * the same name as the question above even though it has the same answer today: one is about how + * elements are built and the other about where bytes come from, and an allocator that changed its + * mind about one of them should not silently change the other. + */ +template +struct allocates_with_operator_new : is_std_allocator {}; + +/** + * @brief Holds the allocator, and holds nothing at all when it has no state. + * + * An svector is meant to be as small as its inline capacity and no larger, and for std::allocator + * and every other stateless allocator it stays that way: the empty base takes no space of its own. + * A stateful allocator is stored, and then the object grows by what it costs, which is the price of + * asking for one. + */ +template && !std::is_final_v> +class allocator_holder : private A { +public: + allocator_holder() = default; + + explicit allocator_holder(A a) + : A(std::move(a)) {} + + [[nodiscard]] auto allocator() -> A& { + return *this; + } + + [[nodiscard]] auto allocator() const -> A const& { + return *this; + } +}; + +template +class allocator_holder { + A m_allocator; + +public: + allocator_holder() = default; + + explicit allocator_holder(A a) + : m_allocator(std::move(a)) {} + + [[nodiscard]] auto allocator() -> A& { + return m_allocator; + } + + [[nodiscard]] auto allocator() const -> A const& { + return m_allocator; + } +}; + +/** + * @brief std::destroy() through the allocator. + * + * The static_cast here and in the five functions like it below says that this branch has no + * use for the allocator, which is a thing gcc and clang work out for themselves -- neither warns + * about a parameter that only the discarded branch of an if constexpr reads. MSVC at /W4 is the one + * this cannot be checked against from here, and the Windows job builds with werror, so the cast + * stays: it costs a line and nothing else, and removing it can only be worth a red build. + */ +template +void alloc_destroy(A& alloc, T* first, T* last) { + if constexpr (builds_elements_directly) { + static_cast(alloc); + std::destroy(first, last); + } else { + for (; first != last; ++first) { + std::allocator_traits::destroy(alloc, first); + } + } +} + +template +void alloc_destroy_n(A& alloc, T* first, size_t n) { + alloc_destroy(alloc, first, first + n); +} + +/** + * @brief Destroys [first, last) on the way out, unless it has been released. + * + * Half built storage can normally say what it holds with a size, and then a destructor is all the + * cleanup anyone needs. This is for the places where the built part is not a prefix of a container, + * so no size can express it. Deleting the copy is what keeps it from being handed around and + * destroying twice. + * + * The allocator is held the way an svector holds it rather than by pointer, so a stateless one adds + * nothing to the guard at all. A pointer would have been a word of its own on a path that was tuned + * to the instruction, and for std::allocator it would have pointed at an empty base that is never + * read. A copy is as good as the original: copying an allocator may not throw, and the copy + * compares equal, which is all destroying through it needs. + */ +template +// NOLINTNEXTLINE(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +class destroy_guard : private allocator_holder { + using T = typename std::allocator_traits::value_type; + using holder = allocator_holder; + + T* m_first; + T* m_last; + +public: + destroy_guard(A const& alloc, T* first, T* last) + : holder(alloc) + , m_first(first) + , m_last(last) {} + + destroy_guard(destroy_guard const&) = delete; + auto operator=(destroy_guard const&) -> destroy_guard& = delete; + + ~destroy_guard() { + alloc_destroy(holder::allocator(), m_first, m_last); + } + + // The guarded range only ever grows, and stays contiguous while it does. + void extend_front(T* first) { + m_first = first; + } + + void extend_back(T* last) { + m_last = last; + } + + void release() { + m_last = m_first; + } +}; + +/** + * @brief Builds n elements at dst, one call to build() each, and destroys whatever it managed to + * build if one of them throws. + * + * Only for an allocator that builds elements its own way: everything else goes through , + * see builds_elements_directly. + */ +template +void construct_each(A& alloc, T* dst, size_t n, Build build) { + auto guard = destroy_guard(alloc, dst, dst); + for (size_t i = 0; i != n; ++i) { + build(dst + i); + guard.extend_back(dst + i + 1); + } + guard.release(); +} + +template +void alloc_uninitialized_copy_n(A& alloc, It first, size_t n, T* dst) { + if constexpr (builds_elements_directly) { + static_cast(alloc); + std::uninitialized_copy_n(first, n, dst); + } else { + construct_each(alloc, dst, n, [&](T* p) { + std::allocator_traits::construct(alloc, p, *first); + ++first; + }); + } +} + +template +void alloc_uninitialized_move(A& alloc, T* first, T* last, T* dst) { + if constexpr (builds_elements_directly) { + static_cast(alloc); + std::uninitialized_move(first, last, dst); + } else { + construct_each(alloc, dst, static_cast(last - first), [&](T* p) { + std::allocator_traits::construct(alloc, p, std::move(*first)); + ++first; + }); + } +} + +template +void alloc_uninitialized_fill_n(A& alloc, T* dst, size_t n, T const& value) { + if constexpr (builds_elements_directly) { + static_cast(alloc); + std::uninitialized_fill_n(dst, n, value); + } else { + construct_each(alloc, dst, n, [&](T* p) { + std::allocator_traits::construct(alloc, p, value); + }); + } +} + +template +void alloc_uninitialized_value_construct_n(A& alloc, T* dst, size_t n) { + if constexpr (builds_elements_directly) { + static_cast(alloc); + std::uninitialized_value_construct_n(dst, n); + } else { + construct_each(alloc, dst, n, [&](T* p) { + std::allocator_traits::construct(alloc, p); + }); + } +} + /** * Holds size & capacity, a glorified struct. */ @@ -125,6 +443,19 @@ public: } }; +/** + * @brief What the allocator is actually asked for: one alignment's worth of bytes. + * + * An allocator hands back memory aligned for its own value_type, and what an indirect svector needs + * is a header followed by an array of T, which is neither. So the rebind target has to state the + * alignment itself. Every T with the same alignment shares this type, and with it one instantiation + * of the rebound allocator. + */ +template +struct alignas(Align) chunk { + std::byte raw[Align]; +}; + /** * @brief Holds header (size+capacity) plus an arbitrary number of T. * @@ -138,6 +469,11 @@ struct storage : public header { static constexpr auto offset_to_data = detail::round_up(sizeof(header), alignment_of_t); static_assert(max_alignment <= __STDCPP_DEFAULT_NEW_ALIGNMENT__); + using chunk_type = chunk; + + template + using rebound = typename std::allocator_traits::template rebind_alloc; + explicit storage(size_t capacity) : header(capacity) {} @@ -146,15 +482,28 @@ struct storage : public header { return std::launder(reinterpret_cast(ptr_to_data)); } + /** + * @brief How many chunks hold a header plus capacity*T. allocator() and dealloc() have to agree on + * this, and all dealloc() has left to work from is the capacity in the header. + */ + [[nodiscard]] static constexpr auto num_chunks(size_t capacity) -> size_t { + return round_up(offset_to_data + sizeof(T) * capacity, max_alignment) / max_alignment; + } + /** * @brief Allocates space for storage plus capacity*T objects. * * Checks to make sure that allocation won't overflow. * + * @param a Allocator to take the memory from, rebound to chunk_type. * @param capacity Number of T to allocate. * @return storage* */ - static auto alloc(size_t capacity) -> storage* { + template + static auto alloc(A& a, size_t capacity) -> storage* { + static_assert(std::is_same_v>::pointer, chunk_type*>, + "sorry, an allocator whose rebound pointer is not a raw pointer is not supported"); + // make sure we don't overflow! auto mem = sizeof(T) * capacity; if (mem < capacity) { @@ -168,24 +517,107 @@ struct storage : public header { throw std::bad_alloc(); } - void* ptr = ::operator new(offset_to_data + sizeof(T) * capacity); + void* ptr = nullptr; + if constexpr (allocates_with_operator_new::value) { + // std::allocator is ::operator new, so this is where it was going anyway. Taking it + // directly is not a shortcut for its own sake: it keeps the byte count already computed + // above instead of dividing it into chunks for allocate() to multiply back, which is + // the only thing an allocator costs a container that just uses the default one. + static_cast(a); + ptr = ::operator new(mem); + } else { + auto chunk_alloc = rebound(a); + ptr = std::allocator_traits>::allocate(chunk_alloc, num_chunks(capacity)); + } if (nullptr == ptr) { - throw std::bad_alloc(); + throw std::bad_alloc(); // LCOV_EXCL_LINE an allocator is supposed to throw rather than return nothing } // use void* to ensure we don't use an overload for T* return new (ptr) storage(capacity); } + + /** + * @brief Counterpart to allocator(). Does not touch the T's, they have to be destroyed already. + */ + template + static void dealloc(A& a, storage* ptr) { + if constexpr (allocates_with_operator_new::value) { + static_cast(a); + std::destroy_at(ptr); + ::operator delete(ptr); + } else { + // read before the header is gone: an allocator wants to be told the size it handed out + auto const n = num_chunks(ptr->capacity()); + std::destroy_at(ptr); + + auto chunk_alloc = rebound(a); + std::allocator_traits>::deallocate(chunk_alloc, reinterpret_cast(ptr), n); + } + } +}; + +/** + * @brief Deallocates storage on the way out, unless it has been released. + * + * For the window in which an allocation exists but no svector holds it yet, which is where an + * insert that has to grow runs the caller's constructors. Deleting the copy is what keeps it from + * being handed around and freeing twice. + * + * Holds the allocator by value for the same reason destroy_guard does, see there. + */ +template +// NOLINTNEXTLINE(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +class storage_guard : private allocator_holder { + using T = typename std::allocator_traits::value_type; + using holder = allocator_holder; + + storage* m_storage; + +public: + storage_guard(A const& alloc, storage* s) + : holder(alloc) + , m_storage(s) {} + + storage_guard(storage_guard const&) = delete; + auto operator=(storage_guard const&) -> storage_guard& = delete; + + ~storage_guard() { + if (m_storage != nullptr) { + storage::dealloc(holder::allocator(), m_storage); + } + } + + [[nodiscard]] auto get() const -> storage* { + return m_storage; + } + + auto release() -> storage* { + auto* s = m_storage; + m_storage = nullptr; + return s; + } }; } // namespace detail -template -class svector { +template > +class svector : private detail::allocator_holder { static_assert(MinInlineCapacity <= 127, "sorry, can't have more than 127 direct elements"); + + using alloc_traits = std::allocator_traits; + using holder = detail::allocator_holder; + + static_assert(std::is_same_v, + "the allocator has to hand out the element type, same as std::vector's does"); + static_assert(std::is_same_v, + "sorry, an allocator with a fancy pointer is not supported: svector's iterator is a plain T*"); + static constexpr auto N = detail::automatic_capacity(MinInlineCapacity); enum class direction { direct, indirect }; + using holder::allocator; + /** * A buffer to hold the data of the svector Depending on direct/indirect mode, the content it holds is like so: * @@ -195,10 +627,49 @@ class svector { * Then 0-X bytes unused (padding), and then the actual inline T data. * indirect: * m_data[0] & 1: lowest bit is 0 for indirect mode - * m_data[0..7]: stores an uintptr_t, which points to the indirect data. + * m_data[0..sizeof(void*)-1]: stores an uintptr_t, which points to the indirect data. + * + * The mode flag and the pointer share m_data[0], which only holds on a little endian target. + * There the pointer's least significant byte lands in m_data[0] and alignment guarantees its + * low bit is 0, so the flag has a byte to live in. On a big endian target m_data[0] would be + * the pointer's most significant byte instead and the flag would read something unrelated. */ alignas(detail::alignment_of_svector()) std::array(MinInlineCapacity)> m_data; + // Nothing in CI can catch this: every runner is little endian. MSVC does not define + // __BYTE_ORDER__ but targets nothing big endian either, so check where a check is possible. +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) + static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, + "svector packs its direct/indirect flag into the low byte of the indirect pointer, " + "which requires a little endian target"); +#endif + + /** + * @brief Whether the whole inline buffer can stand in for the elements it holds. + * + * Only for a trivially copyable T, and only while the buffer is small: such a copy always + * covers the full inline capacity rather than the part in use. do_move_assign() has the + * measurements behind the size limit; swap() rides on the same trade. + * + * An allocator that builds its elements its own way is left out: relocating one bytewise runs + * neither its construct() nor its destroy(), and only the allocator knows whether that is the + * same thing. Every stateless allocator that does not customize those is still in. + */ + static constexpr bool relocate_by_memcpy = + std::is_trivially_copyable_v && detail::builds_elements_directly; + + static constexpr bool relocate_by_copying_m_data = + relocate_by_memcpy && detail::size_of_svector(MinInlineCapacity) <= 128U; + + /** + * @brief Whether destroying an element does anything at all, and so whether it is worth asking. + * + * A trivial destructor is not enough on its own any more: an allocator with a destroy() of its + * own has to see every element go, however little ~T() would have done. + */ + static constexpr bool destroy_is_observable = + !std::is_trivially_destructible_v || !detail::builds_elements_directly; + // direct mode /////////////////////////////////////////////////////////// [[nodiscard]] auto is_direct() const -> bool { @@ -222,6 +693,7 @@ class svector { [[nodiscard]] auto indirect() -> detail::storage* { detail::storage* ptr; // NOLINT(cppcoreguidelines-init-variables) + // NOLINTNEXTLINE(bugprone-sizeof-expression,bugprone-multi-level-implicit-pointer-conversion) std::memcpy(&ptr, m_data.data(), sizeof(ptr)); return ptr; } @@ -230,7 +702,20 @@ class svector { return const_cast(this)->indirect(); // NOLINT(cppcoreguidelines-pro-type-const-cast) } + /** + * @brief set_indirect() without the low bit check, for a pointer already known to be one. + * + * set_indirect() re-reads the byte it just wrote to make sure the mode came out right. Where + * the pointer was just taken off a live indirect svector that is answering a question we have + * already answered, and it is 4 of the 29 instructions of an indirect swap. + */ + void set_indirect_unchecked(detail::storage* ptr) { + // NOLINTNEXTLINE(bugprone-sizeof-expression,bugprone-multi-level-implicit-pointer-conversion) + std::memcpy(m_data.data(), &ptr, sizeof(ptr)); + } + void set_indirect(detail::storage* ptr) { + // NOLINTNEXTLINE(bugprone-sizeof-expression,bugprone-multi-level-implicit-pointer-conversion) std::memcpy(m_data.data(), &ptr, sizeof(ptr)); // safety check to guarantee the lowest bit is 0 @@ -246,19 +731,99 @@ class svector { * * Assumes data is not overlapping */ - static void uninitialized_move_and_destroy(T* source_ptr, T* target_ptr, size_t size) { - if constexpr (std::is_trivially_copyable_v) { + void uninitialized_move_and_destroy(T* source_ptr, T* target_ptr, size_t size) { + // the memcpy runs neither a constructor nor a destructor, so it is only the same thing when + // the allocator would not have run anything of its own either + if constexpr (relocate_by_memcpy) { std::memcpy(target_ptr, source_ptr, size * sizeof(T)); } else { - std::uninitialized_move_n(source_ptr, size, target_ptr); - std::destroy_n(source_ptr, size); + detail::alloc_uninitialized_move(allocator(), source_ptr, source_ptr + size, target_ptr); + detail::alloc_destroy_n(allocator(), source_ptr, size); } } + /** + * @brief True when value is one of our own elements, e.g. from v.push_back(v[0]). + * + * Growing or shifting moves those elements around and assigns over them, so a caller taking a + * T const& has to get it out of the way before that starts. There are two ways to do that and + * only one of them needs this: either copy value to the stack first, which is what the callers + * below do, or build the new element before touching anything, which is what emplace() does. + * insert(pos, value) takes the second route, which is why it does not ask. + * + * Compared as uintptr_t because relational operators on pointers into different objects are + * not specified. + */ + [[nodiscard]] auto is_reference_into_self(T const& value) const -> bool { + auto const p = reinterpret_cast(std::addressof(value)); + auto const first = reinterpret_cast(data()); + return p >= first && p < first + (sizeof(T) * size()); + } + + /** + * @brief Moves all current elements into storage, frees the old one and adopts storage. + * + * Takes ownership of storage even when it fails: T's move constructor is allowed to throw, + * and then nothing here has happened yet, so storage has to be freed again on the way out or + * it leaks. num_constructed says how many elements the caller has already built at the end of + * storage, which then have to be destroyed too. std::uninitialized_move_n already cleans up + * whatever it managed to move itself. + * + * Not a template, so every emplace_back instantiation shares this instead of inlining its + * own copy of the direct/indirect fork. + */ + void take_over_storage(detail::storage* storage, size_t new_size, size_t num_constructed = 0) { + try { + if (is_direct()) { + uninitialized_move_and_destroy(data(), storage->data(), size()); + } else { + uninitialized_move_and_destroy(data(), storage->data(), size()); + detail::storage::dealloc(allocator(), indirect()); + } + } catch (...) { + detail::alloc_destroy_n(allocator(), storage->data() + new_size - num_constructed, num_constructed); + detail::storage::dealloc(allocator(), storage); + throw; + } + storage->size(new_size); + set_indirect(storage); + } + + /** + * @brief Grows and appends in a single step. Precondition: size() == capacity(). + * + * args may reference one of our own elements, as in v.push_back(v[0]). Reallocating first + * would move that element into the new storage and destroy the original, leaving args + * dangling, so the new element is constructed into the fresh storage while the old elements + * are still untouched. Only afterwards are they moved over. + * + * emplace(cend(), ...) relies on this ordering too, don't turn it back into + * reallocate-then-construct. + */ + template + auto emplace_back_grow(size_t s, Args&&... args) -> T& { + // s + 1 > capacity() >= N, so the new storage is always indirect + auto fresh = detail::storage_guard( + allocator(), detail::storage::alloc(allocator(), calculate_new_capacity(s + 1, s))); + + auto* const element = fresh.get()->data() + s; + alloc_traits::construct(allocator(), element, std::forward(args)...); + + // args has been consumed, the old elements can be moved now. take_over_storage() owns the + // allocation from here whether it succeeds or not, so the guard lets go of it first; if the + // move throws, element is the one thing already built in there, hence the 1. + take_over_storage(fresh.release(), s + 1, 1); + return *element; + } + /** * @brief Reallocates all data when capacity changes. * * if new_capacity <= N chooses direct memory, otherwise indirect. + * + * Invalidates every reference into the container, so any T const& argument that might be + * one of our own elements has to be copied out of the way before calling this. See + * is_reference_into_self(). */ void realloc(size_t new_capacity) { if (new_capacity <= N) { @@ -272,24 +837,10 @@ class svector { auto* storage = indirect(); uninitialized_move_and_destroy(storage->data(), direct_data(), storage->size()); set_direct_and_size(storage->size()); - std::destroy_at(storage); - ::operator delete(storage); + detail::storage::dealloc(allocator(), storage); } else { // put everything into indirect storage - auto* storage = detail::storage::alloc(new_capacity); - if (is_direct()) { - // direct -> indirect - uninitialized_move_and_destroy(data(), storage->data(), size()); - storage->size(size()); - } else { - // indirect -> indirect - uninitialized_move_and_destroy(data(), storage->data(), size()); - storage->size(size()); - auto* storage_direct = indirect(); - std::destroy_at(storage_direct); - ::operator delete(storage_direct); - } - set_indirect(storage); + take_over_storage(detail::storage::alloc(allocator(), new_capacity), size()); } } @@ -298,8 +849,10 @@ class svector { */ [[nodiscard]] static auto calculate_new_capacity(size_t size_to_fit, size_t starting_capacity) -> size_t { if (size_to_fit > max_size()) { - // not enough space - throw std::bad_alloc(); + // Asking for more elements than can exist is a size error, not a failure to find + // memory, and std::vector spells it the same way. bad_alloc stays for the case that + // really is one: a size that is legal but that the allocator cannot satisfy. + throw std::length_error("svector: requested size exceeds max_size()"); } if (size_to_fit == 0) { @@ -380,36 +933,70 @@ class svector { } } + /** + * @brief resize(count) for a vector that has just been constructed, so there is nothing to + * shrink and no capacity to keep. + * + * Not resize(count): that one asks whether it has to grow, which here it always does, and gcc + * makes the whole constructor 15 bytes larger for the question. + */ + void resize_to_value_constructed(size_t count) { + reserve(count); + if (is_direct()) { + resize_after_reserve(count); + } else { + resize_after_reserve(count); + } + } + /** * @brief We need variadic arguments so we can either use copy ctor or default ctor */ template + // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) void resize_after_reserve(size_t count, Args&&... args) { auto current_size = size(); if (current_size > count) { - if constexpr (!std::is_trivially_destructible_v) { + if constexpr (destroy_is_observable) { auto* d = data(); - std::destroy(d + count, d + current_size); + detail::alloc_destroy(allocator(), d + count, d + current_size); } } else { - auto* d = data(); - for (auto ptr = d + current_size, end = d + count; ptr != end; ++ptr) { - new (static_cast(ptr)) T(std::forward(args)...); + // The hand written loop this replaces left everything it had already built behind when + // a constructor threw: the size still said current_size, so nothing ever destroyed + // them. These two clean up after themselves, which is the same reason insert() builds + // through them rather than looping. See issue #70. + // + // Value construction, not default construction: resize(n) on an svector has to + // zero the new elements the way T() does. + auto* const first_new = data() + current_size; + if constexpr (sizeof...(Args) == 0) { + detail::alloc_uninitialized_value_construct_n(allocator(), first_new, count - current_size); + } else { + detail::alloc_uninitialized_fill_n(allocator(), first_new, count - current_size, args...); } } set_size(count); } - // Makes sure that to is not past the end iterator + // Makes sure that to is not past the end iterator, and does nothing when that leaves an empty + // range to erase template auto erase_checked_end(T const* cfrom, T const* to) -> T* { auto* const erase_begin = const_cast(cfrom); // NOLINT(cppcoreguidelines-pro-type-const-cast) auto* const container_end = data() + size(); - auto* const erase_end = (std::min)(const_cast(to), container_end); // NOLINT(cppcoreguidelines-pro-type-const-cast) + auto* const erase_end = (std::min)(const_cast(to), container_end); // NOLINT(cppcoreguidelines-pro-type-const-cast) + auto const num_erased = std::distance(erase_begin, erase_end); + + if (num_erased == 0) { + // Not just a shortcut: std::move below would be handed a destination equal to its + // source begin, which it does not allow, and it would self-move-assign every element + // from here to the end. See issue #66. + return erase_begin; + } std::move(erase_end, container_end, erase_begin); - auto const num_erased = std::distance(erase_begin, erase_end); - std::destroy(container_end - num_erased, container_end); + detail::alloc_destroy(allocator(), container_end - num_erased, container_end); set_size(size() - num_erased); return erase_begin; } @@ -431,13 +1018,55 @@ class svector { auto s = std::distance(first, last); reserve(s); - std::uninitialized_copy(first, last, data()); + detail::alloc_uninitialized_copy_n(allocator(), first, static_cast(s), data()); set_size(s); } - // precondition: all uninitialized + /** + * @brief Whether other's allocation, if it has one, could be freed through our allocator. + * + * An allocation only ever goes back to an allocator that compares equal to the one it came + * from, so this is what decides between stealing other's pointer and moving its elements over + * one at a time. Almost always a compile time yes, and then no allocator is even looked at. + */ + [[nodiscard]] auto can_take_over(svector const& other) const -> bool { + if constexpr (alloc_traits::is_always_equal::value) { + static_cast(other); + return true; + } else { + return allocator() == other.allocator(); + } + } + + /** + * @brief Takes over other's elements, and its allocation if it has one. + * + * Precondition: we hold nothing, and our allocator can free what other's allocated -- either + * because they compare equal, or because ours has just been replaced by other's. Every caller + * checks; the one that cannot moves the elements one at a time instead. + */ + // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) void do_move_assign(svector&& other) { - if (!other.is_direct()) { + /** + * Everything that makes up an svector lives inside m_data: in indirect mode just the + * pointer, in direct mode the size byte and the elements themselves. So copying the array + * copies the whole vector, with no branch on the mode and no loop over the elements. See + * issue #54. + * + * Two things have to hold. The elements must survive being relocated bytewise, which is + * what trivially copyable buys: byte-relocating e.g. a libstdc++ std::string in its small + * string mode leaves its data pointer aimed at other's inline buffer, which dangles as + * soon as other is gone. + * + * And the object has to be small, because the copy always covers the full inline capacity + * and not just the used part. Sorting a std::vector of svector holding two elements + * each: gcc 16 -O3 gains 31% at 24 bytes, 21% at 72 bytes, but loses 72% at 264; clang 22 + * gains 38% / 29% with the same crossover. Filling a std::vector by moving svectors into + * it is allocation bound and gains 15% on clang while costing up to 3% on gcc. + */ + if constexpr (relocate_by_copying_m_data) { + m_data = other.m_data; + } else if (!other.is_direct()) { // take other's memory, even when empty set_indirect(other.indirect()); } else { @@ -445,74 +1074,254 @@ class svector { auto s = other.size(); auto* other_end = other_ptr + s; - std::uninitialized_move(other_ptr, other_end, data()); - std::destroy(other_ptr, other_end); + detail::alloc_uninitialized_move(allocator(), other_ptr, other_end, data()); + detail::alloc_destroy(allocator(), other_ptr, other_end); set_size(s); } other.set_direct_and_size(0); } /** - * @brief Shifts data [source_begin, source_end( to the right, starting on target_begin. + * @brief Where insert_n() gets its new elements from: source is a range of count of them. * - * Preconditions: - * * contiguous memory - * * source_begin <= target_begin - * * source_end onwards is uninitialized memory - * - * Destroys then empty elements in [source_begin, source_end( + * construct() builds n of them, starting at the offset'th, into raw storage. assign() puts the + * first n onto elements that are already there. An insert needs both, see insert_n(). */ - static void shift_right(T* source_begin, T* source_end, T* target_begin) { - // 1. uninitialized moves - auto const num_moves = std::distance(source_begin, source_end); - auto const target_end = target_begin + num_moves; - auto const num_uninitialized_move = (std::min)(num_moves, std::distance(source_end, target_end)); - std::uninitialized_move(source_end - num_uninitialized_move, source_end, target_end - num_uninitialized_move); - std::move_backward(source_begin, source_end - num_uninitialized_move, target_end - num_uninitialized_move); - std::destroy(source_begin, (std::min)(source_end, target_begin)); - } + template + struct place_range { + It source; - template - [[nodiscard]] auto make_uninitialized_space_new(size_t s, T* p, size_t count) -> T* { - auto target = svector(); - // we know target is indirect because we're increasing capacity - target.reserve(s + count); - - // move everything [begin, pos[ - auto* target_pos = std::uninitialized_move(data(), p, target.template data()); - - // move everything [pos, end] - std::uninitialized_move(p, data() + s, target_pos + count); - - target.template set_size(s + count); - *this = std::move(target); - return target_pos; - } - - template - [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* { - auto* const p = const_cast(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast) - auto s = size(); - if (s + count > capacity()) { - return make_uninitialized_space_new(s, p, count); + void construct(Allocator& alloc, T* dst, size_t offset, size_t n) const { + detail::alloc_uninitialized_copy_n(alloc, std::next(source, static_cast(offset)), n, dst); } - shift_right(p, data() + s, p + count); - set_size(s + count); + void assign(T* dst, size_t n) const { + std::copy_n(source, n, dst); + } + }; + + /** + * @brief Same, for count copies of one value. + */ + struct place_copies { + T const& value; + + void construct(Allocator& alloc, T* dst, size_t /*offset*/, size_t n) const { + detail::alloc_uninitialized_fill_n(alloc, dst, n, value); + } + + void assign(T* dst, size_t n) const { + std::fill_n(dst, n, value); + } + }; + + static auto place_moved(T& source) -> place_range> { + return {std::make_move_iterator(std::addressof(source))}; + } + + /** + * @brief Inserts count elements at pos, taken from place. Returns the first one. + * + * Invalidates every reference into the container: the elements from pos on are either shifted + * right or moved into a fresh allocation. A T const& argument that might be one of our own + * elements has to be dealt with first, see is_reference_into_self(). + * + * Only the shift below is per (direction, placer) pair; growing goes through commit_grow(), + * which knows nothing about the placer and is one function for all of them. The whole body used + * to be per pair, and four near identical copies of the growth path is what made each additional + * insert form cost about 8 KB where it had been 2.8 KB. See issue #79. + * + * The shift is deliberately not shared as well, and that is the trade the issue is about: as a + * function of its own it is small enough that gcc inlines it straight back, and clang, which + * does not, pays for it. Sharing it costs a compile time count on the paths where there is one + * -- 100 x 1000 emplace(begin()) on svector spends 12.7% more instructions when a one + * element relocation the compiler was unrolling becomes a call to memmove with a runtime length. + * The growth path has no such constant to lose, which is why the seam sits where it does. + * + * There is deliberately no point in here where size() counts memory that holds no element. What + * lands past the old end is constructed, what lands on an element that is still there is + * assigned over it, and size() only ever grows by a step that has already happened. Opening a + * hole and constructing into it afterwards was simpler, but it left the container undestroyable + * for as long as the hole lasted, so every constructor that could throw needed a rollback to + * close it again -- and getting that rollback right is what issues #68 and #74 were about. + * + * What it costs is the strong exception guarantee for an insert that has to assign over live + * elements: failing part way leaves the container with the right number of elements but no + * promise about which, and only the part before pos is certain to be untouched. That is what + * the standard asks of insert, and what std::vector does. The paths that assign over nothing + * are unaffected: growing builds the result in an allocation of its own, and a single element + * goes through emplace(), which builds it before anything is touched. Growing is all or nothing + * only as far as relocating a T is: a move constructor that throws part way through leaves our + * own elements moved from, and then all that is left is that nothing leaks. + */ + template + auto insert_n(T const* pos, size_t count, Place const& place) -> T* { + auto* const p = const_cast(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast) + auto const s = size(); + + if (count == 0) { + // Not just a shortcut: the std::move_backward below would be handed a destination equal + // to its source end, which it does not allow, and it would self-move-assign every + // element it covers. See issue #65. + return p; + } + + // Both written as subtractions so neither can wrap: s <= max_size() and s <= capacity() + // always hold. It used to say s + count > capacity(), which overflowed for a huge count, + // and the wrapped sum then looked small enough to fit, so the in place shift below ran + // straight past the end of the buffer. See issue #69. + if (count > max_size() - s) { + throw std::length_error("svector: requested size exceeds max_size()"); + } + + if (count > capacity() - s) { + return insert_n_new(p, count, place); + } + + auto* const old_end = data() + s; + auto const tail = static_cast(old_end - p); + + if (tail > count) { + // The tail is long enough that shifting it right stays within the old elements plus + // the count raw slots behind them, so all the new elements land on live ones. + detail::alloc_uninitialized_move(allocator(), old_end - count, old_end, old_end); + set_size(s + count); + std::move_backward(p, old_end - count, old_end); + place.assign(p, count); + } else { + // The tail clears the ground it stood on, so the new elements behind the old end have + // nothing under them and are built instead. + place.construct(allocator(), old_end, tail, count - tail); + set_size(s + count - tail); + detail::alloc_uninitialized_move(allocator(), p, old_end, p + count); + set_size(s + count); + place.assign(p, tail); + } return p; } - // makes space for uninitialized data of cout elements. Also updates size. - [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* { + /** + * @brief The other half of insert_n_new(): relocates our elements around the new ones and + * adopts the storage they were built in. Returns the first of them. + * + * Deliberately out of line. This is the bulky part of an insert and the cold one, and having it + * exist once instead of once per (direction, placer) pair is most of what splitting insert_n() + * up buys -- gcc inlines it back into every caller otherwise. See issue #79. + * + * What that costs is one call per reallocation, about 28 instructions, and it is the only thing + * about the split that costs anything: without the attribute a growth-only workload measures + * within 0.05% of the code this replaces, with it svector pays 1.8%. Only a T that + * relocates by memcpy is fast enough for a fixed 28 to show up at all; for std::string the same + * workload is unchanged. That is the trade for ~2 KB per insert form. + */ + template + ANKERL_SVECTOR_NOINLINE auto commit_grow(T* p, size_t count, detail::storage_guard& fresh) -> T* { + auto const s = size(); + auto* const old_data = data(); + auto* const dst = fresh.get()->data(); + auto* const gap = dst + (p - old_data); + + // What is built now is the gap but nothing in front of it, and no size can say that, so for + // as long as the hole lasts the cleanup is spelled out here. Leaving it to the destructor of + // a container that sees a size of zero is what leaked the relocated elements in issue #74. + // Both moves below destroy whatever they managed to build themselves. + auto guard = detail::destroy_guard(allocator(), gap, gap + count); + detail::alloc_uninitialized_move(allocator(), old_data, p, dst); + guard.extend_front(dst); + detail::alloc_uninitialized_move(allocator(), p, old_data + s, gap + count); + guard.release(); + + // Nothing below throws, so this is where the new storage stops being the guard's. + auto* const storage = fresh.release(); + storage->size(s + count); + destroy(); // our elements are all moved from now, and the old allocation can go + set_indirect(storage); + return gap; + } + + /** + * @brief insert_n() for when the elements no longer fit. Builds the result in fresh storage. + * + * The allocation is the guard's until commit_grow() takes it: place.construct() is where the + * caller's constructors run, nothing of ours has moved yet at that point, so an exception there + * means the insert simply has not happened and the storage has to go back. + */ + template + auto insert_n_new(T* p, size_t count, Place const& place) -> T* { + auto const s = size(); + + // the capacity reserve(s + count) on an empty svector would pick, which is where this used + // to start before it built the result itself + auto fresh = detail::storage_guard( + allocator(), detail::storage::alloc(allocator(), calculate_new_capacity(s + count, N))); + auto* const gap = fresh.get()->data() + (p - data()); + + // The new elements before ours: nothing of ours has moved yet if this throws, so the insert + // simply has not happened, and place can still read the elements we hold. + place.construct(allocator(), gap, 0, count); + return commit_grow(p, count, fresh); + } + + template + auto insert_n(T const* pos, size_t count, Place const& place) -> T* { if (is_direct()) { - return make_uninitialized_space(pos, count); + return insert_n(pos, count, place); } - return make_uninitialized_space(pos, count); + return insert_n(pos, count, place); + } + + /** + * @brief swap() for when both hold their elements inline. + * + * Exchanges as far as both of them reach, then hands the remainder of the longer one over. + */ + // uninitialized_move_and_destroy() is the obvious helper for the tail below and is the wrong + // choice: its memcpy specialisation for a trivially copyable T becomes an out of line call, + // which loses to the loop the compiler inlines here. Measured on svector, which + // is trivially copyable and too big for the whole buffer path, so it lands right here: the + // helper is 70 instructions against 157, and 8.93 ns against 5.69. + void swap_direct(svector& other) { + auto const s = direct_size(); + auto const other_s = other.direct_size(); + auto* const mine = direct_data(); + auto* const theirs = other.direct_data(); + auto const common = (std::min)(s, other_s); + + std::swap_ranges(mine, mine + common, theirs); + if (s > other_s) { + detail::alloc_uninitialized_move(allocator(), mine + common, mine + s, theirs + common); + detail::alloc_destroy(allocator(), mine + common, mine + s); + } else { + detail::alloc_uninitialized_move(allocator(), theirs + common, theirs + other_s, mine + common); + detail::alloc_destroy(allocator(), theirs + common, theirs + other_s); + } + + set_direct_and_size(other_s); + other.set_direct_and_size(s); + } + + /** + * @brief swap() for when dir holds its elements inline and ind holds a pointer. + * + * The pointer has to be read out before dir's elements are moved on top of it: when T's + * alignment is below sizeof(void*) the inline storage starts inside the bytes the pointer + * occupies. + */ + static void swap_direct_with_indirect(svector& dir, svector& ind) { + auto* const storage = ind.indirect(); + auto const s = dir.direct_size(); + auto* const from = dir.direct_data(); + + detail::alloc_uninitialized_move(dir.allocator(), from, from + s, ind.direct_data()); + detail::alloc_destroy(dir.allocator(), from, from + s); + + ind.set_direct_and_size(s); + dir.set_indirect(storage); } void destroy() { auto const is_dir = is_direct(); - if constexpr (!std::is_trivially_destructible_v) { + if constexpr (destroy_is_observable) { T* ptr = nullptr; size_t s = 0; if (is_dir) { @@ -522,12 +1331,10 @@ class svector { ptr = data(); s = size(); } - std::destroy_n(ptr, s); + detail::alloc_destroy_n(allocator(), ptr, s); } if (!is_dir) { - auto* storage = indirect(); - std::destroy_at(storage); - ::operator delete(storage); + detail::storage::dealloc(allocator(), indirect()); } set_direct_and_size(0); } @@ -544,6 +1351,7 @@ class svector { public: using value_type = T; + using allocator_type = Allocator; using size_type = size_t; using difference_type = std::ptrdiff_t; using reference = value_type&; @@ -555,23 +1363,41 @@ public: using reverse_iterator = std::reverse_iterator; using const_reverse_iterator = std::reverse_iterator; - svector() { + svector() noexcept(std::is_nothrow_default_constructible_v) { set_direct_and_size(0); } + explicit svector(Allocator const& alloc) noexcept + : holder(alloc) { + set_direct_and_size(0); + } + + /** + * Every constructor that takes an allocator is spelled twice rather than once with an + * `Allocator const& = Allocator()` default argument. A default argument of class type is a + * temporary the caller has to materialize and pass the address of, and it does that even for an + * allocator with nothing in it, so `svector v(n)` grew a stack slot, a lea and a third + * argument register over what it cost before there was an allocator at all -- 39 bytes to 47 on + * clang -Os. The second overload is cheaper than the default argument it replaces. + */ + explicit svector(size_t count) + : svector() { + resize_to_value_constructed(count); + } + + svector(size_t count, Allocator const& alloc) + : svector(alloc) { + resize_to_value_constructed(count); + } + svector(size_t count, T const& value) : svector() { resize(count, value); } - explicit svector(size_t count) - : svector() { - reserve(count); - if (is_direct()) { - resize_after_reserve(count); - } else { - resize_after_reserve(count); - } + svector(size_t count, T const& value, Allocator const& alloc) + : svector(alloc) { + resize(count, value); } template >> @@ -580,27 +1406,106 @@ public: assign(first, last); } - svector(svector const& other) + template >> + svector(InputIt first, InputIt last, Allocator const& alloc) + : svector(alloc) { + assign(first, last); + } + +#if ANKERL_SVECTOR_HAS_RANGES + template R> + svector(std::from_range_t /*unused*/, R&& rg) : svector() { + append_range(std::forward(rg)); + } + + template R> + svector(std::from_range_t /*unused*/, R&& rg, Allocator const& alloc) + : svector(alloc) { + append_range(std::forward(rg)); + } +#endif + + /** + * @brief Copying asks the allocator which one the copy should use. + * + * That is select_on_container_copy_construction(), and for most allocators it hands back the + * same one. An allocator that owns an arena is where it does not: it can say that a copy starts + * from a default constructed one instead of sharing the arena. + * + * It builds the holder from that answer rather than handing it to the constructor below, which + * would be the same materialized temporary the note above is about, on every copy of every + * svector. + */ + svector(svector const& other) + : holder(alloc_traits::select_on_container_copy_construction(other.allocator())) { + set_direct_and_size(0); auto s = other.size(); reserve(s); - std::uninitialized_copy(other.begin(), other.end(), begin()); + detail::alloc_uninitialized_copy_n(allocator(), other.begin(), s, begin()); set_size(s); } - svector(svector&& other) noexcept - : svector() { + svector(svector const& other, Allocator const& alloc) + : svector(alloc) { + auto s = other.size(); + reserve(s); + detail::alloc_uninitialized_copy_n(allocator(), other.begin(), s, begin()); + set_size(s); + } + + /** + * @brief Moving is only noexcept when moving a T is. + * + * std::vector can promise this unconditionally because its move only steals a pointer. In + * direct mode we have to relocate the inline elements instead, which calls T's move + * constructor, so the promise is only ours to make when that one is noexcept. Claiming it + * anyway turns a throwing move into std::terminate, and makes std::move_if_noexcept pick us + * up for a move where it should have fallen back to a copy. See issue #63. + * + * The allocator comes along, so whatever other holds can be taken as it is. Copying one is not + * allowed to throw, so it adds no condition here. + */ + svector(svector&& other) noexcept(std::is_nothrow_move_constructible_v) + : holder(other.allocator()) { + set_direct_and_size(0); do_move_assign(std::move(other)); } + /** + * @brief Moving into a named allocator, which may not be the one other's memory came from. + * + * Then there is nothing to take over: an allocation can only go back to an allocator that + * compares equal to the one that handed it out, so the elements are moved one at a time. + */ + svector(svector&& other, Allocator const& alloc) + : svector(alloc) { + if (can_take_over(other)) { + do_move_assign(std::move(other)); + } else { + assign(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + svector(std::initializer_list init) : svector(init.begin(), init.end()) {} + svector(std::initializer_list init, Allocator const& alloc) + : svector(init.begin(), init.end(), alloc) {} + ~svector() { destroy(); } + // NOLINTNEXTLINE(misc-no-recursion) void assign(size_t count, T const& value) { + if (is_reference_into_self(value)) { + // clear() destroys every element, including the one value refers to. + // Copy it to the stack first, then it's an ordinary assign. v.assign(1000, v[0]) + auto const tmp = value; // NOLINT(performance-unnecessary-copy-initialization) + assign(count, tmp); + return; + } clear(); resize(count, value); } @@ -614,21 +1519,56 @@ public: assign(l.begin(), l.end()); } + /** + * @brief Copies other's elements, and other's allocator when the allocator asks for it. + * + * propagate_on_container_copy_assignment is that ask. What we hold came from the allocator + * that is about to be replaced, and only that one can take it back, so it goes first. + */ auto operator=(svector const& other) -> svector& { if (&other == this) { return *this; } + if constexpr (alloc_traits::propagate_on_container_copy_assignment::value) { + if (!can_take_over(other)) { + destroy(); + } + allocator() = other.allocator(); + } + assign(other.begin(), other.end()); return *this; } - auto operator=(svector&& other) noexcept -> svector& { + /** + * @brief Conditional for the same reason as the move constructor, plus one of its own. + * + * Without propagate_on_container_move_assignment an allocator that does not compare equal to + * other's leaves nothing to steal, and moving the elements across can throw, so the promise is + * only there when it cannot come to that. + */ + auto operator=(svector&& other) noexcept(std::is_nothrow_move_constructible_v && + (alloc_traits::propagate_on_container_move_assignment::value || + alloc_traits::is_always_equal::value)) -> svector& { if (&other == this) { // It doesn't seem to be required to do self-check, but let's do it anyways to be safe return *this; } - destroy(); + + if constexpr (alloc_traits::propagate_on_container_move_assignment::value) { + destroy(); // still ours to free, the allocator that gave it to us is on the next line + allocator() = other.allocator(); + } else if (!can_take_over(other)) { + // Two allocators that do not know about each other's memory: all that is left is to + // move the elements themselves, and other keeps its allocation and its moved from + // elements. The standard asks no more of it than to be usable afterwards. + assign(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + return *this; + } else { + destroy(); + } + do_move_assign(std::move(other)); return *this; } @@ -649,7 +1589,17 @@ public: } } + // NOLINTNEXTLINE(misc-no-recursion) void resize(size_t count, T const& value) { + if (is_reference_into_self(value)) { + // reserve() below moves the elements into new storage and destroys the originals, + // so value has to be copied out of the way first. v.resize(1000, v[0]) + // Deliberately not also testing count > capacity(): duplicating the condition + // below would silently stop matching if that one ever changes. + auto const tmp = value; // NOLINT(performance-unnecessary-copy-initialization) + resize(count, tmp); + return; + } if (count > capacity()) { reserve(count); } @@ -668,60 +1618,64 @@ public: } } - [[nodiscard]] auto capacity() const -> size_t { + [[nodiscard]] auto capacity() const noexcept -> size_t { if (is_direct()) { return capacity(); } return capacity(); } - [[nodiscard]] auto size() const -> size_t { + [[nodiscard]] auto size() const noexcept -> size_t { if (is_direct()) { return size(); } return size(); } - [[nodiscard]] auto data() -> T* { + [[nodiscard]] auto data() noexcept -> T* { if (is_direct()) { return direct_data(); } return indirect()->data(); } - [[nodiscard]] auto data() const -> T const* { + [[nodiscard]] auto data() const noexcept -> T const* { return const_cast(this)->data(); // NOLINT(cppcoreguidelines-pro-type-const-cast) } + /** + * @brief Appends one element, growing first if there is no room. + * + * Written as two whole paths rather than one path that asks which mode it is in three times. + * Constructing the element writes through a T*, which the compiler cannot prove does not alias + * our own bytes, so every is_direct() after it is a fresh load and every indirect() after it is + * a fresh pointer chase. Deciding once and carrying the answer in a register is worth about a + * third of the instructions of an append. + */ template auto emplace_back(Args&&... args) -> T& { - size_t c; // NOLINT(cppcoreguidelines-init-variables) - size_t s; // NOLINT(cppcoreguidelines-init-variables) - bool is_dir = is_direct(); - if (is_dir) { - c = capacity(); - s = size(); - } else { - c = capacity(); - s = size(); + if (is_direct()) { + auto const s = direct_size(); + if (s != N) { + // construct before recording the size, so a throwing constructor does not leave + // the vector claiming an element that was never built + auto* const element = direct_data() + s; + alloc_traits::construct(allocator(), element, std::forward(args)...); + set_direct_and_size(s + 1); + return *element; + } + return emplace_back_grow(s, std::forward(args)...); } - if (s == c) { - auto new_capacity = calculate_new_capacity(s + 1, c); - realloc(new_capacity); - // reallocation happened, so we definitely are now in indirect mode - is_dir = false; + auto* const storage = indirect(); + auto const s = storage->size(); + if (s != storage->capacity()) { + auto* const element = storage->data() + s; + alloc_traits::construct(allocator(), element, std::forward(args)...); + storage->size(s + 1); + return *element; } - - T* ptr; // NOLINT(cppcoreguidelines-init-variables) - if (is_dir) { - ptr = data() + s; - set_size(s + 1); - } else { - ptr = data() + s; - set_size(s + 1); - } - return *new (static_cast(ptr)) T(std::forward(args)...); + return emplace_back_grow(s, std::forward(args)...); } void push_back(T const& value) { @@ -732,11 +1686,11 @@ public: emplace_back(std::move(value)); } - [[nodiscard]] auto operator[](size_t idx) const -> T const& { + [[nodiscard]] auto operator[](size_t idx) const noexcept -> T const& { return *(data() + idx); } - [[nodiscard]] auto operator[](size_t idx) -> T& { + [[nodiscard]] auto operator[](size_t idx) noexcept -> T& { return *(data() + idx); } @@ -751,79 +1705,79 @@ public: return const_cast(this)->at(idx); // NOLINT(cppcoreguidelines-pro-type-const-cast) } - [[nodiscard]] auto begin() const -> T const* { + [[nodiscard]] auto begin() const noexcept -> T const* { return data(); } - [[nodiscard]] auto cbegin() const -> T const* { + [[nodiscard]] auto cbegin() const noexcept -> T const* { return begin(); } - [[nodiscard]] auto begin() -> T* { + [[nodiscard]] auto begin() noexcept -> T* { return data(); } - [[nodiscard]] auto end() -> T* { + [[nodiscard]] auto end() noexcept -> T* { if (is_direct()) { return data() + size(); } return data() + size(); } - [[nodiscard]] auto end() const -> T const* { + [[nodiscard]] auto end() const noexcept -> T const* { return const_cast(this)->end(); // NOLINT(cppcoreguidelines-pro-type-const-cast) } - [[nodiscard]] auto cend() const -> T const* { + [[nodiscard]] auto cend() const noexcept -> T const* { return end(); } - [[nodiscard]] auto rbegin() -> reverse_iterator { + [[nodiscard]] auto rbegin() noexcept -> reverse_iterator { return reverse_iterator{end()}; } - [[nodiscard]] auto rbegin() const -> const_reverse_iterator { + [[nodiscard]] auto rbegin() const noexcept -> const_reverse_iterator { return crbegin(); } - [[nodiscard]] auto crbegin() const -> const_reverse_iterator { + [[nodiscard]] auto crbegin() const noexcept -> const_reverse_iterator { return const_reverse_iterator{end()}; } - [[nodiscard]] auto rend() -> reverse_iterator { + [[nodiscard]] auto rend() noexcept -> reverse_iterator { return reverse_iterator{begin()}; } - [[nodiscard]] auto rend() const -> const_reverse_iterator { + [[nodiscard]] auto rend() const noexcept -> const_reverse_iterator { return crend(); } - [[nodiscard]] auto crend() const -> const_reverse_iterator { + [[nodiscard]] auto crend() const noexcept -> const_reverse_iterator { return const_reverse_iterator{begin()}; } - [[nodiscard]] auto front() const -> T const& { + [[nodiscard]] auto front() const noexcept -> T const& { return *data(); } - [[nodiscard]] auto front() -> T& { + [[nodiscard]] auto front() noexcept -> T& { return *data(); } - [[nodiscard]] auto back() -> T& { + [[nodiscard]] auto back() noexcept -> T& { if (is_direct()) { return *(data() + size() - 1); } return *(data() + size() - 1); } - [[nodiscard]] auto back() const -> T const& { + [[nodiscard]] auto back() const noexcept -> T const& { return const_cast(this)->back(); // NOLINT(cppcoreguidelines-pro-type-const-cast) } - void clear() { - if constexpr (!std::is_trivially_destructible_v) { - std::destroy(begin(), end()); + void clear() noexcept { + if constexpr (destroy_is_observable) { + detail::alloc_destroy(allocator(), begin(), end()); } if (is_direct()) { @@ -833,11 +1787,11 @@ public: } } - [[nodiscard]] auto empty() const -> bool { + [[nodiscard]] auto empty() const noexcept -> bool { return 0U == size(); } - void pop_back() { + void pop_back() noexcept { if (is_direct()) { pop_back(); } else { @@ -845,13 +1799,81 @@ public: } } - [[nodiscard]] static auto max_size() -> size_t { - return (std::numeric_limits::max)(); + /** + * @brief Deliberately not min()'d with the allocator's own max_size(). + * + * An allocator with a smaller limit of its own still holds: asking it for more than it has + * throws out of allocate(), which is where a container that had min()'d would have ended up + * anyway. What that limit cannot do is be read without an allocator to read it from, and this + * function has been public and static since before there was one -- svector::max_size() + * is spelled that way in test/unit/insert.cpp. Making it a non-static member is a breaking + * change and belongs to a major version, not to adding an allocator. + * + * Divided by sizeof(T) because a count is not a byte count. It used to answer PTRDIFF_MAX for + * every T, which claimed a size no svector could reach: alloc() refuses anything whose bytes + * pass PTRDIFF_MAX, so the real ceiling has always been this. std::vector answers the same. + */ + [[nodiscard]] static auto max_size() noexcept -> size_t { + return static_cast((std::numeric_limits::max)()) / sizeof(T); } - void swap(svector& other) { - // TODO we could try to do the minimum number of moves - std::swap(*this, other); + [[nodiscard]] auto get_allocator() const noexcept -> Allocator { + return allocator(); + } + + /** + * @brief Exchanges the contents with other. + * + * std::swap(*this, other) would do it in three whole container moves, and in direct mode a + * container move is every element moved and the original destroyed. Two indirect svectors need + * not touch an element at all, and a mixed pair only has to relocate the inline side. + * + * Measured against the three move version, gcc 16 -O3, inline capacity 7, ns per swap: + * two heap std::string 22.9 -> 15.5, unequal sizes 23.0 -> 16.6, uint64_t 2.9 -> 2.3, + * indirect 1.7 -> 1.3, mixed 15.0 -> 11.3. One case is worse: two equal length runs of short + * strings, 28.5 -> 34.8, because std::swap_ranges finds std::string::swap, and for a string + * small enough to live inside itself that is three copies of the internal buffer where a + * relocation would have been one. That is libstdc++'s trade, not one this can pick around, and + * it buys the other six. + * + * The condition is what the work below actually needs: relocating between the two inline + * buffers is a move construction, and exchanging elements in place is a swap. + * + * The allocators are exchanged too when propagate_on_container_swap says so. When it does not + * and they do not compare equal the standard says the behaviour is undefined, and this makes no + * attempt to be nice about it: the two allocations would simply change hands. + */ + void swap(svector& other) noexcept(std::is_nothrow_move_constructible_v && std::is_nothrow_swappable_v && + (alloc_traits::propagate_on_container_swap::value || + alloc_traits::is_always_equal::value)) { + if (this == &other) { + return; + } + + if constexpr (alloc_traits::propagate_on_container_swap::value) { + using std::swap; + swap(allocator(), other.allocator()); + } + + auto const is_dir = is_direct(); + auto const other_is_dir = other.is_direct(); + + if (!is_dir && !other_is_dir) { + // both on the heap, so nothing but the two pointers moves + auto* const mine = indirect(); + set_indirect_unchecked(other.indirect()); + other.set_indirect_unchecked(mine); + } else if constexpr (relocate_by_copying_m_data) { + // m_data is the whole of an svector whichever mode it is in, so for a T that can be + // relocated by copying bytes this exchanges everything at once, and vectorizes + std::swap(m_data, other.m_data); + } else if (is_dir && other_is_dir) { + swap_direct(other); + } else if (is_dir) { + swap_direct_with_indirect(*this, other); + } else { + swap_direct_with_indirect(other, *this); + } } void shrink_to_fit() { @@ -874,10 +1896,28 @@ public: template auto emplace(const_iterator pos, Args&&... args) -> iterator { - auto* p = make_uninitialized_space(pos, 1); - return new (static_cast(p)) T(std::forward(args)...); + if (pos == cend()) { + // no elements have to move out of the way, and emplace_back already builds the + // new element before it grows, so args referencing us is fine there + return std::addressof(emplace_back(std::forward(args)...)); + } + + // args may reference one of our own elements, and making space either shifts those + // elements right or moves them into a new allocation, either of which leaves args + // dangling. Build the element first. Inserting in the middle already moves every + // element after pos, so one extra move does not change the cost. + // + // tmp is a plain local and not built through the allocator, which only shows for an + // allocator that hands its elements something, e.g. a std::pmr::string built here holds the + // default resource until it is moved into place. What ends up in the container is + // constructed from it through the allocator, so it holds the right one; what it costs is + // that the move is a copy when the two resources differ. + auto tmp = T(std::forward(args)...); + return insert_n(pos, 1, place_moved(tmp)); } + // Both of these want the element built before anything is shifted, which is what emplace() + // does, so they go there rather than say it again. See insert_n() for what that buys. auto insert(const_iterator pos, T const& value) -> iterator { return emplace(pos, value); } @@ -886,10 +1926,15 @@ public: return emplace(pos, std::move(value)); } + // NOLINTNEXTLINE(misc-no-recursion) auto insert(const_iterator pos, size_t count, T const& value) -> iterator { - auto* p = make_uninitialized_space(pos, count); - std::uninitialized_fill_n(p, count, value); - return p; + if (is_reference_into_self(value)) { + // the shift moves our elements around and assigns over them, so value has to be + // copied out of the way first. v.insert(v.begin(), 1000, v[0]) + auto const tmp = value; // NOLINT(performance-unnecessary-copy-initialization) + return insert(pos, count, tmp); + } + return insert_n(pos, count, place_copies{value}); } template @@ -908,15 +1953,14 @@ public: return begin() + s; } - auto tmp = svector(first, last); + auto tmp = svector(first, last, allocator()); return insert(pos, std::make_move_iterator(tmp.begin()), std::make_move_iterator(tmp.end())); } template auto insert(const_iterator pos, It first, It last, std::forward_iterator_tag /*unused*/) -> iterator { - auto* p = make_uninitialized_space(pos, std::distance(first, last)); - std::uninitialized_copy(first, last, p); - return p; + auto const count = static_cast(std::distance(first, last)); + return insert_n(pos, count, place_range{first}); } template >> @@ -924,10 +1968,95 @@ public: return insert(pos, first, last, typename std::iterator_traits::iterator_category()); } +#if ANKERL_SVECTOR_HAS_RANGES + /** + * @brief The C++23 range members, which std::vector has and this did not. + * + * All of them go through the iterator pair members, so a range gets the same growth, the same + * exception guarantees and the same self referencing checks an iterator pair already got, + * rather than a second implementation of all three. + * + * A range cannot always be handed over as a pair: its sentinel need not be its iterator, and + * its iterator need not publish an iterator_category. One that cannot is built into a + * temporary first. That costs an allocation for exactly the ranges that could not have been + * sized anyway. + */ + template + static constexpr bool is_iterator_pair_range = + std::ranges::common_range && detail::has_iterator_category>::value; + + template R> + auto insert_range(const_iterator pos, R&& rg) -> iterator { + if constexpr (is_iterator_pair_range) { + return insert(pos, std::ranges::begin(rg), std::ranges::end(rg)); + } else { + auto tmp = svector(allocator()); + for (auto&& element : rg) { + tmp.emplace_back(std::forward(element)); + } + return insert(pos, std::make_move_iterator(tmp.begin()), std::make_move_iterator(tmp.end())); + } + } + + template R> + void append_range(R&& rg) { + static_cast(insert_range(cend(), std::forward(rg))); + } + + template R> + void assign_range(R&& rg) { + if constexpr (is_iterator_pair_range) { + assign(std::ranges::begin(rg), std::ranges::end(rg)); + } else { + clear(); + append_range(std::forward(rg)); + } + } +#endif + auto insert(const_iterator pos, std::initializer_list l) -> iterator { return insert(pos, l.begin(), l.end()); } + /** + * @brief Resizes to count elements, letting op initialize the new ones in place. + * + * Same contract as std::string::resize_and_overwrite, see + * https://en.cppreference.com/w/cpp/string/basic_string/resize_and_overwrite + * + * op is called as op(p, count) with p == data(), and returns the actual new size: + * * p[0, min(count, size())) are the existing elements, readable and assignable. + * * p[min(count, size()), count) is raw uninitialized storage. op has to construct + * every element it wants to keep, e.g. with placement new. + * * op returns r, which must be in [0, count]. Afterwards size() == r, so p[0, r) + * must all be constructed objects when op returns. + * + * This skips the value-initialization that resize() would do, which is what makes it + * faster: for e.g. reading into an svector the zero fill is pure overhead. + */ + template + void resize_and_overwrite(size_t count, Operation op) { + // step 1: make room. This preserves the existing elements and may switch to indirect mode. + reserve(count); + + auto const old_size = size(); + if (count < old_size) { + // Shrinking: the tail is gone. Commit the smaller size *before* running op, so that + // if op throws, the destructor sees exactly the elements that are still alive. + detail::alloc_destroy_n(allocator(), data() + count, old_size - count); + set_size(count); + } + + // step 2: op initializes [min(count, old_size), count) and tells us how much it kept. + // The stored size is still min(count, old_size) here, so an exception escaping op + // destroys the untouched prefix and leaks only what op itself constructed. + auto const new_size = std::move(op)(data(), count); + + // step 3: commit. new_size <= count <= capacity() is a precondition, so in direct mode + // this can never overflow the 7 bit size field. + set_size(new_size); + } + auto erase(const_iterator pos) -> iterator { return erase(pos, pos + 1); } @@ -940,36 +2069,47 @@ public: } }; -template -[[nodiscard]] auto operator==(svector const& a, svector const& b) -> bool { +template +[[nodiscard]] auto operator==(svector const& a, svector const& b) -> bool { return std::equal(a.begin(), a.end(), b.begin(), b.end()); } -template -[[nodiscard]] auto operator!=(svector const& a, svector const& b) -> bool { +template +[[nodiscard]] auto operator!=(svector const& a, svector const& b) -> bool { return !(a == b); } -template -[[nodiscard]] auto operator<(svector const& a, svector const& b) -> bool { +template +[[nodiscard]] auto operator<(svector const& a, svector const& b) -> bool { return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); } -template -[[nodiscard]] auto operator>=(svector const& a, svector const& b) -> bool { +template +[[nodiscard]] auto operator>=(svector const& a, svector const& b) -> bool { return !(a < b); } -template -[[nodiscard]] auto operator>(svector const& a, svector const& b) -> bool { +template +[[nodiscard]] auto operator>(svector const& a, svector const& b) -> bool { return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end()); } -template -[[nodiscard]] auto operator<=(svector const& a, svector const& b) -> bool { +template +[[nodiscard]] auto operator<=(svector const& a, svector const& b) -> bool { return !(a > b); } +/** + * @brief Found by argument dependent lookup, so std::swap and everything built on it get the + * member swap rather than the generic three move version. + * + * Only for two svectors of the same inline capacity; the generic one cannot exchange those either. + */ +template +void swap(svector& a, svector& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); +} + } // namespace ANKERL_SVECTOR_NAMESPACE } // namespace ankerl @@ -977,16 +2117,16 @@ namespace std { // NOLINTNEXTLINE(cert-dcl58-cpp) inline namespace ANKERL_SVECTOR_NAMESPACE { -template -constexpr auto erase(ankerl::svector& sv, U const& value) -> typename ankerl::svector::size_type { +template +constexpr auto erase(ankerl::svector& sv, U const& value) -> typename ankerl::svector::size_type { auto* removed_begin = std::remove(sv.begin(), sv.end(), value); auto num_removed = std::distance(removed_begin, sv.end()); sv.erase(removed_begin, sv.end()); return num_removed; } -template -constexpr auto erase_if(ankerl::svector& sv, Pred pred) -> typename ankerl::svector::size_type { +template +constexpr auto erase_if(ankerl::svector& sv, Pred pred) -> typename ankerl::svector::size_type { auto* removed_begin = std::remove_if(sv.begin(), sv.end(), pred); auto num_removed = std::distance(removed_begin, sv.end()); sv.erase(removed_begin, sv.end()); diff --git a/include/eepp/thirdparty/unordered_dense.h b/include/eepp/thirdparty/unordered_dense.h index 97ef720d9..0ac177b9a 100644 --- a/include/eepp/thirdparty/unordered_dense.h +++ b/include/eepp/thirdparty/unordered_dense.h @@ -1,7 +1,7 @@ ///////////////////////// ankerl::unordered_dense::{map, set} ///////////////////////// -// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion. -// Version 4.8.1 +// A fast & densely stored hashmap and hashset. +// Version 5.0.1 // https://github.com/martinus/unordered_dense // // Licensed under the MIT License . @@ -30,8 +30,8 @@ #define ANKERL_UNORDERED_DENSE_H // see https://semver.org/spec/v2.0.0.html -#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes -#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 8 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality +#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 5 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes +#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality #define ANKERL_UNORDERED_DENSE_VERSION_PATCH 1 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes // API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ @@ -66,8 +66,58 @@ #endif #ifdef _MSC_VER # define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline) +# define ANKERL_UNORDERED_DENSE_FORCEINLINE __forceinline #else # define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline)) +# define ANKERL_UNORDERED_DENSE_FORCEINLINE inline __attribute__((always_inline)) +#endif + +// Data prefetch hint, a no-op where there is nothing to spell it with. MSVC has no +// __builtin_prefetch and used to get the no-op, which quietly cost it the one the probe issues for +// a group's value indices -- measured at 3 cycles off every hit, so a whole compiler was paying for +// a missing spelling. Both MSVC intrinsics come from , which is included further down; +// that is in time, because a macro needs its declarations where it is expanded and every expansion +// is inside the table. Taken from boost, which covers the same three cases. +#if defined(__GNUC__) || defined(__clang__) +# define ANKERL_UNORDERED_DENSE_PREFETCH(addr) __builtin_prefetch(addr) // NOLINT(cppcoreguidelines-macro-usage) +#elif defined(_MSC_VER) && (defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_PREFETCH(addr) _mm_prefetch(reinterpret_cast(addr), _MM_HINT_T0) +#elif defined(_MSC_VER) && defined(_M_ARM64) +# define ANKERL_UNORDERED_DENSE_PREFETCH(addr) __prefetch(addr) // NOLINT(cppcoreguidelines-macro-usage) +#else +# define ANKERL_UNORDERED_DENSE_PREFETCH(addr) static_cast(addr) // NOLINT(cppcoreguidelines-macro-usage) +#endif + +// SSE2 is part of the x86-64 baseline, so comparing a group's sixteen fingerprints in one +// instruction is available on every x86-64 build without asking for it. Elsewhere, and in a build +// that defines this to 0, they are compared eight per machine word with ordinary arithmetic. +// +// This picks the code, never the layout: a group is sixteen slots either way, so two translation +// units that disagree about this macro -- which they may, it is documented as a per-target switch +// -- still agree about every byte of the index they share. +#if !defined(ANKERL_UNORDERED_DENSE_HAS_SSE2) +# if defined(__SSE2__) || (defined(_MSC_VER) && (defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2))) +# define ANKERL_UNORDERED_DENSE_HAS_SSE2 1 // NOLINT(cppcoreguidelines-macro-usage) +# else +# define ANKERL_UNORDERED_DENSE_HAS_SSE2 0 // NOLINT(cppcoreguidelines-macro-usage) +# endif +#endif + +// The same sixteen-at-once compare on AArch64, which is the other baseline worth having: NEON is +// mandatory there, so this needs no runtime dispatch either. Restricted to little endian because +// the mask below reads the comparison result as one 64 bit word, and to AArch64 because 32 bit ARM +// lacks the horizontal ops -- both fall back to SWAR, which is correct everywhere. +#if !defined(ANKERL_UNORDERED_DENSE_HAS_NEON) +# if defined(__ARM_NEON) && defined(__aarch64__) && \ + (!defined(__BYTE_ORDER__) || !defined(__ORDER_BIG_ENDIAN__) || (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__)) +# define ANKERL_UNORDERED_DENSE_HAS_NEON 1 // NOLINT(cppcoreguidelines-macro-usage) +# else +# define ANKERL_UNORDERED_DENSE_HAS_NEON 0 // NOLINT(cppcoreguidelines-macro-usage) +# endif +#endif +#if ANKERL_UNORDERED_DENSE_HAS_SSE2 && ANKERL_UNORDERED_DENSE_HAS_NEON +# error "ANKERL_UNORDERED_DENSE_HAS_SSE2 and ANKERL_UNORDERED_DENSE_HAS_NEON cannot both be on" #endif #if defined(__clang__) && defined(__has_attribute) @@ -93,6 +143,15 @@ # if !ANKERL_UNORDERED_DENSE_STD_MODULE # include "stl.h" # endif +# if ANKERL_UNORDERED_DENSE_HAS_SSE2 +# include // for _mm_loadu_si128, _mm_cmpeq_epi8, ... +# endif +# if ANKERL_UNORDERED_DENSE_HAS_NEON +# include // for vld1q_u8, vceqq_u8, vshrn_n_u16, ... +# endif +# if defined(_MSC_VER) +# include // for _BitScanForward +# endif # if __has_cpp_attribute(likely) && __has_cpp_attribute(unlikely) && ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L # define ANKERL_UNORDERED_DENSE_LIKELY_ATTR [[likely]] // NOLINT(cppcoreguidelines-macro-usage) @@ -131,6 +190,9 @@ namespace detail { [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() { throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements"); } +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_changed() { + throw std::logic_error("ankerl::unordered_dense: an element's key changed after it was inserted; use replace_key()"); +} # else @@ -143,17 +205,51 @@ namespace detail { [[noreturn]] inline void on_error_too_many_elements() { abort(); } +[[noreturn]] inline void on_error_key_changed() { + abort(); +} # endif +// Index of the lowest set bit, for the lane mask a group compare produces. x +// must not be zero. +[[nodiscard]] inline auto countr_zero(std::uint32_t x) -> unsigned { +# if defined(_MSC_VER) + unsigned long idx{}; + _BitScanForward(&idx, x); + return static_cast(idx); +# else + return static_cast(__builtin_ctz(x)); +# endif +} + +# if ANKERL_UNORDERED_DENSE_HAS_NEON +// NEON's match mask is one bit per lane four bits apart, so it needs the whole word. +[[nodiscard]] inline auto countr_zero(std::uint64_t x) -> unsigned { +# if defined(_MSC_VER) + unsigned long idx{}; + _BitScanForward64(&idx, x); + return static_cast(idx); +# else + return static_cast(__builtin_ctzll(x)); +# endif +} +# endif + } // namespace detail // hash /////////////////////////////////////////////////////////////////////// -// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash -// No big-endian support (because different values on different machines don't matter), -// hardcodes seed and the secret, reformats the code, and clang-tidy fixes. -namespace detail::wyhash { +// This is no longer wyhash and does not produce wyhash's values, so it is not named after it. It is +// descended from it: https://github.com/wangyi-fudan/wyhash gives the reads, the multiply-and-xor +// mix, the short path and the chained lanes for long keys. What changed is the middle lengths, +// restructured into independent blocks, which the comment on hash_bytes() explains. If it ever +// leaves this header as something callers can use on its own it will be called `ankerlhash`; until +// then the entry points are `detail::hash_bytes` and `detail::hash_int` and the name is not needed. +// +// No big-endian support, because different values on different machines do not matter here. +// The seed and the secret are hardcoded, so there is nothing to salt a table with. +namespace detail::hash_impl { inline void mum(std::uint64_t* a, std::uint64_t* b) { # if defined(__SIZEOF_INT128__) @@ -208,11 +304,38 @@ inline void mum(std::uint64_t* a, std::uint64_t* b) { return (static_cast(p[0]) << 16U) | (static_cast(p[k >> 1U]) << 8U) | p[k - 1]; } -[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, std::size_t len) -> std::uint64_t { - static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f), - UINT64_C(0xe7037ed1a0b428db), - UINT64_C(0x8ebc6af09c88c6e3), - UINT64_C(0x589965cc75374cc3)}; +// The shape of this is wyhash's up to 16 bytes and for anything past 144, and in between it is +// not: every 16 byte block is mixed on its own, with its own pair of secrets, and the results are +// xor-folded into one finalizer. wyhash chains the blocks through `seed`, so a 48 byte key is +// three multiplies one after another and then the finalizer, and a map lookup waits for all of +// them before it can so much as form the group address. Here the block multiplies are independent, +// so the latency of any key up to 144 bytes is one multiply plus the finalizer, and the block +// loop's trip count -- a data-dependent branch that mispredicts whenever lengths vary -- is a +// short chain of compares that the predictor learns from the top. The last sixteen bytes are +// always a block of their own, wherever they fall, so every byte is read at least once and nothing +// is read past the end. +// +// Measured on the scored benchmark's own keys (8 to 135 bytes, skewed short), one function per +// binary, ns per hash: throughput 2.52 to 2.00 under clang and 2.18 to 2.05 under gcc, latency +// 8.64 to 7.69 and 8.47 to 7.81, with fewer branch misses on both. Two shapes measured and +// rejected on the way: making the block range branchless by always mixing three (17-48) or six +// (49-96) overlapping blocks, which costs more in redundant multiplies than it saves in +// mispredictions; and a one multiply short path, which fails an avalanche test outright at 8 bytes +// (output bits that never flip for some input bits), as does dropping the finalizer in the block +// range. Both multiplies stay. +// +// Independent blocks need distinct secrets: with a shared one, swapping two blocks gives the same +// hash. Sixteen pairs cover 144 bytes, and past that the chained lanes take over, where reuse is +// harmless because the chain carries the position. The secrets have wyhash's property, every +// byte with four bits set, odd, and were drawn once from a fixed seed. +[[maybe_unused]] [[nodiscard]] inline auto hash_bytes(void const* key, std::size_t len) -> std::uint64_t { + static constexpr auto secret = std::array{ + UINT64_C(0xa0761d6478bd642f), UINT64_C(0xe7037ed1a0b428db), UINT64_C(0x8ebc6af09c88c6e3), UINT64_C(0x589965cc75374cc3), + UINT64_C(0x2d358dccaa6c78a5), UINT64_C(0x8bb84b93962eacc9), UINT64_C(0x4b33a62ed433d4a3), UINT64_C(0xa693c93927d87217), + UINT64_C(0x2b63728e53473c2b), UINT64_C(0x696cb2a95635a3c5), UINT64_C(0xa9ccd81ed1b29359), UINT64_C(0x5c2d66ace48db84d), + UINT64_C(0x69a99c5c53b4ca2d), UINT64_C(0x9a9c5a1b27d10f69), UINT64_C(0x2b27f02dc3d4360f), UINT64_C(0x2b39665c8d2d5553), + UINT64_C(0x966cd8878bb4b187), UINT64_C(0xc6351e99932b1ee1), UINT64_C(0xd1c5d24d63c959c9), UINT64_C(0x56c54d9c955aca2b), + UINT64_C(0xd136d27872563559)}; auto const* p = static_cast(key); std::uint64_t seed = secret[0]; @@ -220,54 +343,204 @@ inline void mum(std::uint64_t* a, std::uint64_t* b) { std::uint64_t b{}; if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) ANKERL_UNORDERED_DENSE_LIKELY_ATTR { - if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) + if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 8)) ANKERL_UNORDERED_DENSE_LIKELY_ATTR { - a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U)); - b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U)); + // two (potentially overlapping) 8 byte reads cover the whole input + a = r8(p); + b = r8(p + len - 8); } - else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) + else if (len >= 4) { + a = r4(p); + b = r4(p + len - 4); + } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) ANKERL_UNORDERED_DENSE_LIKELY_ATTR { + // b stays zero: r3 packs all len bytes it is given into a, and there are at + // most three of them. a = r3(p, len); - b = 0; } - else { - a = 0; - b = 0; - } + // ... and an empty input needs no branch of its own: it hashes whatever a and b were + // declared with, which is the zero it has to be. Assigning it again here is what a + // deletion sweep of this file kept pointing at. + + // Return, rather than falling through to the same expression at the end of the + // function. Falling through makes seed, a and b values of two paths at once, and then + // the compiler cannot fold the constant seed of this one into the mix: measured, the + // short path costs 36 instructions that way and 24 this way. + return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); } - else { - std::size_t i = len; - if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) - ANKERL_UNORDERED_DENSE_UNLIKELY_ATTR { - std::uint64_t see1 = seed; - std::uint64_t see2 = seed; - do { - seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); - see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1); - see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2); - p += 48; - i -= 48; - } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48)); - seed ^= see1 ^ see2; + + if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 144)) + ANKERL_UNORDERED_DENSE_LIKELY_ATTR { + // The first block and the last sixteen bytes, then whole blocks from the front for as + // long as there are any: a key of 17 to 32 bytes is two multiplies, one of 129 to 144 + // is nine, all of them independent. + auto x = + mix(r8(p) ^ secret[1], r8(p + 8) ^ secret[2]) ^ mix(r8(p + len - 16) ^ secret[3], r8(p + len - 8) ^ secret[4]); + if (len > 32) { + x ^= mix(r8(p + 16) ^ secret[5], r8(p + 24) ^ secret[6]); + if (len > 48) { + x ^= mix(r8(p + 32) ^ secret[7], r8(p + 40) ^ secret[8]); + if (len > 64) { + x ^= mix(r8(p + 48) ^ secret[9], r8(p + 56) ^ secret[10]); + if (len > 80) { + x ^= mix(r8(p + 64) ^ secret[11], r8(p + 72) ^ secret[12]); + if (len > 96) { + x ^= mix(r8(p + 80) ^ secret[13], r8(p + 88) ^ secret[14]); + if (len > 112) { + x ^= mix(r8(p + 96) ^ secret[15], r8(p + 104) ^ secret[16]); + if (len > 128) { + x ^= mix(r8(p + 112) ^ secret[17], r8(p + 120) ^ secret[18]); + } + } + } + } + } + } } - while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) - ANKERL_UNORDERED_DENSE_UNLIKELY_ATTR { - seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); - i -= 16; - p += 16; - } - a = r8(p + i - 16); - b = r8(p + i - 8); + return mix(secret[1] ^ len, x); + } + + // Anything longer, in chained lanes of 16 bytes, ending on the same expression as above. + std::size_t i = len; + std::uint64_t see1 = seed; + std::uint64_t see2 = seed; + // Six lanes cost three more accumulators to set up and fold back in, so the block has to + // run more than once to pay for them. Entering it at 96 meant exactly one iteration for + // everything from 97 to 192 bytes, which never can: measured, 23.4 cycles for a 100 byte key + // against 21.8 when it takes the 48 byte loop instead, and 29.3 against 28.2 at 150. Above + // 192 the block runs at least twice and wins again -- 143.5 cycles against 147.1 at 1000 + // bytes -- so it keeps those. + if (i > 192) { + // 6 independent lanes: twice the instruction level parallelism of the 48 byte loop below + std::uint64_t see3 = seed; + std::uint64_t see4 = seed; + std::uint64_t see5 = seed; + do { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1); + see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2); + see3 = mix(r8(p + 48) ^ secret[4], r8(p + 56) ^ see3); + see4 = mix(r8(p + 64) ^ secret[5], r8(p + 72) ^ see4); + see5 = mix(r8(p + 80) ^ secret[6], r8(p + 88) ^ see5); + p += 96; + i -= 96; + } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 96)); + seed ^= see3 ^ see4 ^ see5; + } + while (i > 48) { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1); + see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2); + p += 48; + i -= 48; + } + seed ^= see1 ^ see2; + while (i > 16) { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + i -= 16; + p += 16; } - return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); + // the tail lane only depends on the input, not on seed, so it can execute in parallel + // with the lane loops above, and a single dependent mix finishes the hash + auto tail = mix(r8(p + i - 16) ^ secret[2], r8(p + i - 8) ^ secret[3]); + return mix(secret[1] ^ len, seed ^ tail); } -[[nodiscard]] inline auto hash(std::uint64_t x) -> std::uint64_t { - return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15)); +[[nodiscard]] inline auto hash_int(std::uint64_t x) -> std::uint64_t { + return mix(x, UINT64_C(0x9E3779B97F4A7C15)); } -} // namespace detail::wyhash +} // namespace detail::hash_impl + +namespace detail { + +// The two entry points, at `detail` scope because that is where callers writing their own hash for +// their own type reach for them, and doc/usage.md shows exactly that. +using hash_impl::hash_bytes; +using hash_impl::hash_int; + +} // namespace detail + +namespace detail { + +struct nonesuch {}; + +template class Op, class... Args> +struct detector { + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> { + using value_t = std::true_type; + using type = Op; +}; + +template