Slavasil/shortener-bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial

Browse Source
This commit is contained in:
Slavasil
2024-10-10 19:05:48 +00:00
commit cffdcba6af
1880 changed files with 813614 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

426
td/tdutils/CMakeLists.txt Normal file
View File

@@ -0,0 +1,426 @@
if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2"))
message(FATAL_ERROR "CMake >= 3.0.2 is required")
endif()
option(TDUTILS_MIME_TYPE "Generate MIME types conversion; requires gperf" ON)
option(TDUTILS_USE_EXTERNAL_DEPENDENCIES "Use external libraries if available" ON)
if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
set(CMAKE_INSTALL_LIBDIR "lib")
endif()
if (NOT ZLIB_FOUND AND TDUTILS_USE_EXTERNAL_DEPENDENCIES)
find_package(ZLIB)
endif()
if (ZLIB_FOUND)
set(TD_HAVE_ZLIB 1)
message(STATUS "Found ZLIB: ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}")
# OpenSSL internally depends on zlib
if (NOT OPENSSL_FOUND)
find_package(OpenSSL)
endif()
if (OPENSSL_FOUND)
set(TD_HAVE_OPENSSL 1)
endif()
endif()
if (NOT CRC32C_FOUND AND TDUTILS_USE_EXTERNAL_DEPENDENCIES)
find_package(Crc32c QUIET)
endif()
if (CRC32C_FOUND)
set(TD_HAVE_CRC32C 1)
endif()
if (TD_WITH_ABSEIL AND TDUTILS_USE_EXTERNAL_DEPENDENCIES)
find_package(ABSL QUIET)
if (ABSL_FOUND)
set(TD_HAVE_ABSL 1)
endif()
endif()
configure_file(td/utils/config.h.in td/utils/config.h @ONLY)
add_subdirectory(generate)
set_source_files_properties(${TDMIME_AUTO} PROPERTIES GENERATED TRUE)
if (CLANG OR GCC)
set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-conversion")
elseif (MSVC)
set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4267")
endif()
if (CLANG)
set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-deprecated-register")
endif()
set(TDUTILS_SOURCE
td/utils/port/Clocks.cpp
td/utils/port/FileFd.cpp
td/utils/port/IPAddress.cpp
td/utils/port/MemoryMapping.cpp
td/utils/port/path.cpp
td/utils/port/platform.cpp
td/utils/port/PollFlags.cpp
td/utils/port/rlimit.cpp
td/utils/port/ServerSocketFd.cpp
td/utils/port/signals.cpp
td/utils/port/sleep.cpp
td/utils/port/SocketFd.cpp
td/utils/port/stacktrace.cpp
td/utils/port/Stat.cpp
td/utils/port/StdStreams.cpp
td/utils/port/thread_local.cpp
td/utils/port/UdpSocketFd.cpp
td/utils/port/uname.cpp
td/utils/port/user.cpp
td/utils/port/wstring_convert.cpp
td/utils/port/detail/Epoll.cpp
td/utils/port/detail/EventFdBsd.cpp
td/utils/port/detail/EventFdLinux.cpp
td/utils/port/detail/EventFdWindows.cpp
td/utils/port/detail/Iocp.cpp
td/utils/port/detail/KQueue.cpp
td/utils/port/detail/NativeFd.cpp
td/utils/port/detail/Poll.cpp
td/utils/port/detail/Select.cpp
td/utils/port/detail/ThreadIdGuard.cpp
td/utils/port/detail/ThreadPthread.cpp
td/utils/port/detail/WineventPoll.cpp
${TDMIME_AUTO}
td/utils/AsyncFileLog.cpp
td/utils/base64.cpp
td/utils/BigNum.cpp
td/utils/buffer.cpp
td/utils/BufferedUdp.cpp
td/utils/check.cpp
td/utils/crypto.cpp
td/utils/emoji.cpp
td/utils/ExitGuard.cpp
td/utils/FileLog.cpp
td/utils/filesystem.cpp
td/utils/find_boundary.cpp
td/utils/FlatHashTable.cpp
td/utils/FloodControlGlobal.cpp
td/utils/Gzip.cpp
td/utils/GzipByteFlow.cpp
td/utils/Hints.cpp
td/utils/HttpDate.cpp
td/utils/HttpUrl.cpp
td/utils/JsonBuilder.cpp
td/utils/logging.cpp
td/utils/misc.cpp
td/utils/MpmcQueue.cpp
td/utils/OptionParser.cpp
td/utils/PathView.cpp
td/utils/Random.cpp
td/utils/SharedSlice.cpp
td/utils/Slice.cpp
td/utils/StackAllocator.cpp
td/utils/Status.cpp
td/utils/StringBuilder.cpp
td/utils/tests.cpp
td/utils/Time.cpp
td/utils/Timer.cpp
td/utils/tl_parsers.cpp
td/utils/translit.cpp
td/utils/TsCerr.cpp
td/utils/TsFileLog.cpp
td/utils/TsLog.cpp
td/utils/unicode.cpp
td/utils/utf8.cpp
td/utils/port/Clocks.h
td/utils/port/config.h
td/utils/port/CxCli.h
td/utils/port/EventFd.h
td/utils/port/EventFdBase.h
td/utils/port/FileFd.h
td/utils/port/FromApp.h
td/utils/port/IoSlice.h
td/utils/port/IPAddress.h
td/utils/port/MemoryMapping.h
td/utils/port/Mutex.h
td/utils/port/path.h
td/utils/port/platform.h
td/utils/port/Poll.h
td/utils/port/PollBase.h
td/utils/port/PollFlags.h
td/utils/port/rlimit.h
td/utils/port/RwMutex.h
td/utils/port/ServerSocketFd.h
td/utils/port/signals.h
td/utils/port/sleep.h
td/utils/port/SocketFd.h
td/utils/port/stacktrace.h
td/utils/port/Stat.h
td/utils/port/StdStreams.h
td/utils/port/thread.h
td/utils/port/thread_local.h
td/utils/port/UdpSocketFd.h
td/utils/port/uname.h
td/utils/port/user.h
td/utils/port/wstring_convert.h
td/utils/port/detail/Epoll.h
td/utils/port/detail/EventFdBsd.h
td/utils/port/detail/EventFdLinux.h
td/utils/port/detail/EventFdWindows.h
td/utils/port/detail/Iocp.h
td/utils/port/detail/KQueue.h
td/utils/port/detail/NativeFd.h
td/utils/port/detail/Poll.h
td/utils/port/detail/PollableFd.h
td/utils/port/detail/Select.h
td/utils/port/detail/skip_eintr.h
td/utils/port/detail/ThreadIdGuard.h
td/utils/port/detail/ThreadPthread.h
td/utils/port/detail/ThreadStl.h
td/utils/port/detail/WineventPoll.h
td/utils/AesCtrByteFlow.h
td/utils/algorithm.h
td/utils/as.h
td/utils/AsyncFileLog.h
td/utils/AtomicRead.h
td/utils/base64.h
td/utils/benchmark.h
td/utils/BigNum.h
td/utils/bits.h
td/utils/buffer.h
td/utils/BufferedFd.h
td/utils/BufferedReader.h
td/utils/BufferedUdp.h
td/utils/ByteFlow.h
td/utils/CancellationToken.h
td/utils/ChainScheduler.h
td/utils/ChangesProcessor.h
td/utils/check.h
td/utils/Closure.h
td/utils/CombinedLog.h
td/utils/common.h
td/utils/ConcurrentHashTable.h
td/utils/Container.h
td/utils/Context.h
td/utils/crypto.h
td/utils/DecTree.h
td/utils/Destructor.h
td/utils/emoji.h
td/utils/Enumerator.h
td/utils/EpochBasedMemoryReclamation.h
td/utils/ExitGuard.h
td/utils/FileLog.h
td/utils/filesystem.h
td/utils/find_boundary.h
td/utils/fixed_vector.h
td/utils/FlatHashMap.h
td/utils/FlatHashMapChunks.h
td/utils/FlatHashSet.h
td/utils/FlatHashTable.h
td/utils/FloodControlFast.h
td/utils/FloodControlGlobal.h
td/utils/FloodControlStrict.h
td/utils/format.h
td/utils/Gzip.h
td/utils/GzipByteFlow.h
td/utils/Hash.h
td/utils/HashMap.h
td/utils/HashSet.h
td/utils/HashTableUtils.h
td/utils/HazardPointers.h
td/utils/Heap.h
td/utils/Hints.h
td/utils/HttpDate.h
td/utils/HttpUrl.h
td/utils/int_types.h
td/utils/invoke.h
td/utils/JsonBuilder.h
td/utils/List.h
td/utils/logging.h
td/utils/MapNode.h
td/utils/MemoryLog.h
td/utils/misc.h
td/utils/MovableValue.h
td/utils/MpmcQueue.h
td/utils/MpmcWaiter.h
td/utils/MpscLinkQueue.h
td/utils/MpscPollableQueue.h
td/utils/Named.h
td/utils/NullLog.h
td/utils/ObjectPool.h
td/utils/Observer.h
td/utils/optional.h
td/utils/OptionParser.h
td/utils/OrderedEventsProcessor.h
td/utils/overloaded.h
td/utils/Parser.h
td/utils/PathView.h
td/utils/Promise.h
td/utils/queue.h
td/utils/Random.h
td/utils/ScopeGuard.h
td/utils/SetNode.h
td/utils/SharedObjectPool.h
td/utils/SharedSlice.h
td/utils/Slice-decl.h
td/utils/Slice.h
td/utils/SliceBuilder.h
td/utils/Span.h
td/utils/SpinLock.h
td/utils/StackAllocator.h
td/utils/Status.h
td/utils/StealingQueue.h
td/utils/Storer.h
td/utils/StorerBase.h
td/utils/StringBuilder.h
td/utils/tests.h
td/utils/ThreadLocalStorage.h
td/utils/ThreadSafeCounter.h
td/utils/Time.h
td/utils/TimedStat.h
td/utils/Timer.h
td/utils/tl_helpers.h
td/utils/tl_parsers.h
td/utils/tl_storers.h
td/utils/TlDowncastHelper.h
td/utils/TlStorerToString.h
td/utils/translit.h
td/utils/TsCerr.h
td/utils/TsFileLog.h
td/utils/TsList.h
td/utils/TsLog.h
td/utils/type_traits.h
td/utils/UInt.h
td/utils/uint128.h
td/utils/unicode.h
td/utils/unique_ptr.h
td/utils/unique_value_ptr.h
td/utils/utf8.h
td/utils/Variant.h
td/utils/VectorQueue.h
td/utils/WaitFreeHashMap.h
td/utils/WaitFreeHashSet.h
td/utils/WaitFreeVector.h
)
if (TDUTILS_MIME_TYPE)
set(TDUTILS_SOURCE
${TDUTILS_SOURCE}
td/utils/MimeType.cpp
td/utils/MimeType.h
)
endif()
set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/bitmask.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/ChainScheduler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/emoji.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/HashSet.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/HttpUrl.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/List.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/log.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/OptionParser.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/StealingQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/WaitFreeHashMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/WaitFreeHashSet.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/WaitFreeVector.cpp
PARENT_SCOPE
)
add_library(tdutils STATIC ${TDUTILS_SOURCE})
if (NOT CMAKE_CROSSCOMPILING AND TDUTILS_MIME_TYPE)
add_dependencies(tdutils tdmime_auto)
endif()
if (DEFINED CMAKE_THREAD_LIBS_INIT)
target_link_libraries(tdutils PUBLIC ${CMAKE_THREAD_LIBS_INIT})
endif()
target_include_directories(tdutils PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
if (OPENSSL_FOUND)
target_link_libraries(tdutils PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES})
target_include_directories(tdutils SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
if (WIN32)
if (MINGW)
target_link_libraries(tdutils PRIVATE ws2_32 mswsock crypt32)
else()
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Crypt32)
endif()
endif()
endif()
if (ZLIB_FOUND)
target_link_libraries(tdutils PRIVATE ${ZLIB_LIBRARIES})
target_include_directories(tdutils SYSTEM PRIVATE ${ZLIB_INCLUDE_DIR})
endif()
if (CRC32C_FOUND)
target_link_libraries(tdutils PRIVATE crc32c)
endif()
if (ABSL_FOUND)
target_link_libraries(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash)
endif()
if (WIN32)
# find_library for system libraries doesn't work for UWP builds
# find_library(WS2_32_LIBRARY ws2_32)
# find_library(MSWSOCK_LIBRARY Mswsock)
# target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY})
if (MINGW)
target_link_libraries(tdutils PRIVATE ws2_32 mswsock normaliz psapi)
else()
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz psapi)
endif()
if (NOT CMAKE_SYSTEM_NAME STREQUAL "WindowsStore")
target_link_libraries(tdutils PRIVATE shell32)
endif()
endif()
if (ANDROID)
target_link_libraries(tdutils PRIVATE log)
endif()
find_package(Atomics REQUIRED)
if (ATOMICS_LIBRARIES)
target_link_libraries(tdutils PUBLIC "${ATOMICS_LIBRARIES}")
endif()
install(TARGETS tdutils EXPORT TdStaticTargets
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
)
if (TD_TEST_FOLLY AND ABSL_FOUND AND TDUTILS_USE_EXTERNAL_DEPENDENCIES)
find_package(benchmark QUIET)
find_package(folly QUIET)
find_package(gflags QUIET)
if (benchmark_FOUND AND folly_FOUND)
add_executable(benchmark-hashset test/hashset_benchmark.cpp)
target_link_libraries(benchmark-hashset PRIVATE tdutils benchmark::benchmark Folly::folly absl::flat_hash_map absl::hash)
endif()
endif()

70
td/tdutils/generate/CMakeLists.txt Normal file
View File

@@ -0,0 +1,70 @@
if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2"))
message(FATAL_ERROR "CMake >= 3.0.2 is required")
endif()
# Generates files for MIME type <-> extension conversions
# DEPENDS ON: gperf grep
if (NOT TDUTILS_MIME_TYPE)
return()
endif()
file(MAKE_DIRECTORY auto)
set(TDMIME_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp
${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp
)
set(TDMIME_AUTO
${TDMIME_SOURCE}
PARENT_SCOPE
)
add_custom_target(tdmime_auto DEPENDS ${TDMIME_SOURCE})
if (NOT CMAKE_CROSSCOMPILING)
find_program(GPERF_EXECUTABLE gperf)
if (NOT GPERF_EXECUTABLE)
message(FATAL_ERROR "Could NOT find gperf. Add path to gperf executable to PATH environment variable or specify it manually using GPERF_EXECUTABLE option, i.e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"<path to gperf executable>\"'.")
endif()
set(GPERF_FILES
${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.gperf
${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.gperf
)
set(GPERF_GEN_SOURCE generate_mime_types_gperf.cpp)
add_executable(generate_mime_types_gperf ${GPERF_GEN_SOURCE})
add_custom_command(
OUTPUT ${GPERF_FILES}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND generate_mime_types_gperf mime_types.txt ${GPERF_FILES}
DEPENDS generate_mime_types_gperf mime_types.txt
)
if (CMAKE_HOST_WIN32)
set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/mime_type_to_extension.cpp auto/mime_type_to_extension.gperf)
else()
set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 auto/mime_type_to_extension.gperf | grep -v __gnu_inline__ > auto/mime_type_to_extension.cpp)
endif()
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND ${MIME_TYPE_TO_EXTENSION_CMD}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.gperf
)
if (CMAKE_HOST_WIN32)
set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/extension_to_mime_type.cpp auto/extension_to_mime_type.gperf)
else()
set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 auto/extension_to_mime_type.gperf | grep -v __gnu_inline__ > auto/extension_to_mime_type.cpp)
endif()
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND ${EXTENSION_TO_MIME_TYPE_CMD}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.gperf
)
endif()

156
td/tdutils/generate/generate_mime_types_gperf.cpp Normal file
View File

@@ -0,0 +1,156 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <fstream>
#include <iostream>
#include <map>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
static std::pair<std::string, std::string> split(std::string s, char delimiter = ' ') {
auto delimiter_pos = s.find(delimiter);
if (delimiter_pos == std::string::npos) {
return {std::move(s), ""};
} else {
auto head = s.substr(0, delimiter_pos);
auto tail = s.substr(delimiter_pos + 1);
return {head, tail};
}
}
static bool generate(const char *file_name, const char *from_name, const char *to_name,
const std::map<std::string, std::string> &map) {
// binary mode is needed for MSYS2 gperf
std::ofstream out(file_name, std::ios_base::trunc | std::ios_base::binary);
if (!out) {
std::cerr << "Can't open output file \"" << file_name << std::endl;
return false;
}
out << "%struct-type\n";
out << "%ignore-case\n";
out << "%language=ANSI-C\n";
out << "%readonly-tables\n";
out << "%includes\n";
out << "%enum\n";
out << "%define slot-name " << from_name << "\n";
out << "%define initializer-suffix ,nullptr\n";
out << "%define slot-name " << from_name << "\n";
out << "%define hash-function-name " << from_name << "_hash\n";
out << "%define lookup-function-name search_" << from_name << "\n";
// out << "%define class-name " << from_name << "_to_" << to_name << "\n";
out << "struct " << from_name << "_and_" << to_name << " {\n";
out << " const char *" << from_name << ";\n";
out << " const char *" << to_name << ";\n";
out << "}\n";
out << "%%\n";
for (auto &value : map) {
out << '"' << value.first << "\", \"" << value.second << '"' << "\n";
}
out << "%%\n";
out << "const char *" << from_name << "_to_" << to_name << "(const char *" << from_name << ", size_t " << from_name
<< "_len) {\n";
out << " const auto &result = search_" << from_name << "(" << from_name << ", " << from_name << "_len);\n";
out << " if (result == nullptr) {\n";
out << " return nullptr;\n";
out << " }\n";
out << "\n";
out << " return result->" << to_name << ";\n";
out << "}\n";
return true;
}
static bool is_private_mime_type(const std::string &mime_type) {
return mime_type.find("/x-") != std::string::npos;
}
int main(int argc, char *argv[]) {
if (argc != 4) {
std::cerr << "Wrong number of arguments supplied. Expected 'generate_mime_types_gperf <mime_types.txt> "
"<mime_type_to_extension.cpp> <extension_to_mime_type.cpp>'"
<< std::endl;
return EXIT_FAILURE;
}
std::ifstream mime_types_file(argv[1]);
if (!mime_types_file) {
std::cerr << "Can't open input file \"" << argv[1] << std::endl;
return EXIT_FAILURE;
}
std::map<std::string, std::string> mime_type_to_extension;
std::map<std::string, std::string> extension_to_mime_type;
std::string line;
while (std::getline(mime_types_file, line)) {
while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) {
line.pop_back();
}
std::string mime_type;
std::string extensions_string;
std::tie(mime_type, extensions_string) = split(line, '\t');
if (mime_type.empty()) {
std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl;
continue;
}
auto extensions_start_position = extensions_string.find_first_not_of(" \t");
if (extensions_start_position == std::string::npos) {
std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl;
continue;
}
extensions_string = extensions_string.substr(extensions_start_position);
std::vector<std::string> extensions;
while (!extensions_string.empty()) {
extensions.emplace_back();
std::tie(extensions.back(), extensions_string) = split(extensions_string);
}
assert(!extensions.empty());
std::map<std::string, std::string> preffered_extensions{{"image/jpeg", "jpg"}, {"audio/mpeg", "mp3"},
{"audio/midi", "midi"}, {"text/x-pascal", "pas"},
{"text/x-asm", "asm"}, {"video/quicktime", "mov"}};
std::size_t index = 0;
if (preffered_extensions.count(mime_type) != 0) {
index = std::find(extensions.begin(), extensions.end(), preffered_extensions[mime_type]) - extensions.begin();
assert(index < extensions.size());
}
if (mime_type_to_extension.emplace_hint(mime_type_to_extension.end(), mime_type, extensions[index])->second !=
extensions[index]) {
std::cerr << "MIME type \"" << mime_type << "\" has more than one extensions list" << std::endl;
}
for (auto &extension : extensions) {
if (!extension_to_mime_type.emplace(extension, mime_type).second) {
if (is_private_mime_type(extension_to_mime_type[extension]) == is_private_mime_type(mime_type)) {
std::cerr << "Extension \"" << extension << "\" matches more than one type" << std::endl;
} else {
if (!is_private_mime_type(mime_type)) {
extension_to_mime_type[extension] = mime_type;
}
}
}
}
}
if (!generate(argv[2], "mime_type", "extension", mime_type_to_extension)) {
return EXIT_FAILURE;
}
if (!generate(argv[3], "extension", "mime_type", extension_to_mime_type)) {
return EXIT_FAILURE;
}
}

779
td/tdutils/generate/mime_types.txt Normal file
View File

@@ -0,0 +1,779 @@
application/andrew-inset ez
application/applixware aw
application/atom+xml atom
application/atomcat+xml atomcat
application/atomsvc+xml atomsvc
application/ccxml+xml ccxml
application/cdmi-capability cdmia
application/cdmi-container cdmic
application/cdmi-domain cdmid
application/cdmi-object cdmio
application/cdmi-queue cdmiq
application/cu-seeme cu
application/davmount+xml davmount
application/docbook+xml dbk
application/dssc+der dssc
application/dssc+xml xdssc
application/ecmascript es
application/emma+xml emma
application/epub+zip epub
application/exi exi
application/font-tdpfr pfr
application/gml+xml gml
application/gpx+xml gpx
application/gxf gxf
application/hyperstudio stk
application/inkml+xml ink inkml
application/ipfix ipfix
application/java-archive jar
application/java-serialized-object ser
application/java-vm class
application/javascript js
application/json json
application/jsonml+json jsonml
application/lost+xml lostxml
application/mac-binhex40 hqx
application/mac-compactpro cpt
application/mads+xml mads
application/marc mrc
application/marcxml+xml mrcx
application/mathematica ma nb mb
application/mathml+xml mathml
application/mbox mbox
application/mediaservercontrol+xml mscml
application/metalink+xml metalink
application/metalink4+xml meta4
application/mets+xml mets
application/mods+xml mods
application/mp21 m21 mp21
application/mp4 mp4s
application/msword doc dot
application/mxf mxf
application/octet-stream bin dms lrf mar so dist distz pkg bpk dump elc deploy
application/oda oda
application/oebps-package+xml opf
application/ogg ogx
application/omdoc+xml omdoc
application/onenote onetoc onetoc2 onetmp onepkg
application/oxps oxps
application/patch-ops-error+xml xer
application/pdf pdf
application/pgp-encrypted pgp
application/pgp-signature asc sig
application/pics-rules prf
application/pkcs10 p10
application/pkcs7-mime p7m p7c
application/pkcs7-signature p7s
application/pkcs8 p8
application/pkix-attr-cert ac
application/pkix-cert cer
application/pkix-crl crl
application/pkix-pkipath pkipath
application/pkixcmp pki
application/pls+xml pls
application/postscript ai eps ps
application/prs.cww cww
application/pskc+xml pskcxml
application/rdf+xml rdf
application/reginfo+xml rif
application/relax-ng-compact-syntax rnc
application/resource-lists+xml rl
application/resource-lists-diff+xml rld
application/rls-services+xml rs
application/rpki-ghostbusters gbr
application/rpki-manifest mft
application/rpki-roa roa
application/rsd+xml rsd
application/rss+xml rss
application/rtf rtf
application/sbml+xml sbml
application/scvp-cv-request scq
application/scvp-cv-response scs
application/scvp-vp-request spq
application/scvp-vp-response spp
application/sdp sdp
application/set-payment-initiation setpay
application/set-registration-initiation setreg
application/shf+xml shf
application/smil+xml smi smil
application/sparql-query rq
application/sparql-results+xml srx
application/srgs gram
application/srgs+xml grxml
application/sru+xml sru
application/ssdl+xml ssdl
application/ssml+xml ssml
application/tei+xml tei teicorpus
application/thraud+xml tfi
application/timestamped-data tsd
application/vnd.3gpp.pic-bw-large plb
application/vnd.3gpp.pic-bw-small psb
application/vnd.3gpp.pic-bw-var pvb
application/vnd.3gpp2.tcap tcap
application/vnd.3m.post-it-notes pwn
application/vnd.accpac.simply.aso aso
application/vnd.accpac.simply.imp imp
application/vnd.acucobol acu
application/vnd.acucorp atc acutc
application/vnd.adobe.air-application-installer-package+zip air
application/vnd.adobe.formscentral.fcdt fcdt
application/vnd.adobe.fxp fxp fxpl
application/vnd.adobe.xdp+xml xdp
application/vnd.adobe.xfdf xfdf
application/vnd.ahead.space ahead
application/vnd.airzip.filesecure.azf azf
application/vnd.airzip.filesecure.azs azs
application/vnd.amazon.ebook azw
application/vnd.americandynamics.acc acc
application/vnd.amiga.ami ami
application/vnd.android.package-archive apk
application/vnd.anser-web-certificate-issue-initiation cii
application/vnd.anser-web-funds-transfer-initiation fti
application/vnd.antix.game-component atx
application/vnd.apple.installer+xml mpkg
application/vnd.apple.mpegurl m3u8
application/vnd.aristanetworks.swi swi
application/vnd.astraea-software.iota iota
application/vnd.audiograph aep
application/vnd.blueice.multipass mpm
application/vnd.bmi bmi
application/vnd.businessobjects rep
application/vnd.chemdraw+xml cdxml
application/vnd.chipnuts.karaoke-mmd mmd
application/vnd.cinderella cdy
application/vnd.claymore cla
application/vnd.cloanto.rp9 rp9
application/vnd.clonk.c4group c4g c4d c4f c4p c4u
application/vnd.cluetrust.cartomobile-config c11amc
application/vnd.cluetrust.cartomobile-config-pkg c11amz
application/vnd.commonspace csp
application/vnd.contact.cmsg cdbcmsg
application/vnd.cosmocaller cmc
application/vnd.crick.clicker clkx
application/vnd.crick.clicker.keyboard clkk
application/vnd.crick.clicker.palette clkp
application/vnd.crick.clicker.template clkt
application/vnd.crick.clicker.wordbank clkw
application/vnd.criticaltools.wbs+xml wbs
application/vnd.ctc-posml pml
application/vnd.cups-ppd ppd
application/vnd.curl.car car
application/vnd.curl.pcurl pcurl
application/vnd.dart dart
application/vnd.data-vision.rdz rdz
application/vnd.dece.data uvf uvvf uvd uvvd
application/vnd.dece.ttml+xml uvt uvvt
application/vnd.dece.unspecified uvx uvvx
application/vnd.dece.zip uvz uvvz
application/vnd.denovo.fcselayout-link fe_launch
application/vnd.dna dna
application/vnd.dolby.mlp mlp
application/vnd.dpgraph dpg
application/vnd.dreamfactory dfac
application/vnd.ds-keypoint kpxx
application/vnd.dvb.ait ait
application/vnd.dvb.service svc
application/vnd.dynageo geo
application/vnd.ecowin.chart mag
application/vnd.enliven nml
application/vnd.epson.esf esf
application/vnd.epson.msf msf
application/vnd.epson.quickanime qam
application/vnd.epson.salt slt
application/vnd.epson.ssf ssf
application/vnd.eszigno3+xml es3 et3
application/vnd.ezpix-album ez2
application/vnd.ezpix-package ez3
application/vnd.fdf fdf
application/vnd.fdsn.mseed mseed
application/vnd.fdsn.seed seed dataless
application/vnd.flographit gph
application/vnd.fluxtime.clip ftc
application/vnd.framemaker fm frame maker book
application/vnd.frogans.fnc fnc
application/vnd.frogans.ltf ltf
application/vnd.fsc.weblaunch fsc
application/vnd.fujitsu.oasys oas
application/vnd.fujitsu.oasys2 oa2
application/vnd.fujitsu.oasys3 oa3
application/vnd.fujitsu.oasysgp fg5
application/vnd.fujitsu.oasysprs bh2
application/vnd.fujixerox.ddd ddd
application/vnd.fujixerox.docuworks xdw
application/vnd.fujixerox.docuworks.binder xbd
application/vnd.fuzzysheet fzs
application/vnd.genomatix.tuxedo txd
application/vnd.geogebra.file ggb
application/vnd.geogebra.tool ggt
application/vnd.geometry-explorer gex gre
application/vnd.geonext gxt
application/vnd.geoplan g2w
application/vnd.geospace g3w
application/vnd.gmx gmx
application/vnd.google-earth.kml+xml kml
application/vnd.google-earth.kmz kmz
application/vnd.grafeq gqf gqs
application/vnd.groove-account gac
application/vnd.groove-help ghf
application/vnd.groove-identity-message gim
application/vnd.groove-injector grv
application/vnd.groove-tool-message gtm
application/vnd.groove-tool-template tpl
application/vnd.groove-vcard vcg
application/vnd.hal+xml hal
application/vnd.handheld-entertainment+xml zmm
application/vnd.hbci hbci
application/vnd.hhe.lesson-player les
application/vnd.hp-hpgl hpgl
application/vnd.hp-hpid hpid
application/vnd.hp-hps hps
application/vnd.hp-jlyt jlt
application/vnd.hp-pcl pcl
application/vnd.hp-pclxl pclxl
application/vnd.hydrostatix.sof-data sfd-hdstx
application/vnd.ibm.minipay mpy
application/vnd.ibm.modcap afp listafp list3820
application/vnd.ibm.rights-management irm
application/vnd.ibm.secure-container sc
application/vnd.iccprofile icc icm
application/vnd.igloader igl
application/vnd.immervision-ivp ivp
application/vnd.immervision-ivu ivu
application/vnd.insors.igm igm
application/vnd.intercon.formnet xpw xpx
application/vnd.intergeo i2g
application/vnd.intu.qbo qbo
application/vnd.intu.qfx qfx
application/vnd.ipunplugged.rcprofile rcprofile
application/vnd.irepository.package+xml irp
application/vnd.is-xpr xpr
application/vnd.isac.fcs fcs
application/vnd.jam jam
application/vnd.jcp.javame.midlet-rms rms
application/vnd.jisp jisp
application/vnd.joost.joda-archive joda
application/vnd.kahootz ktz ktr
application/vnd.kde.karbon karbon
application/vnd.kde.kchart chrt
application/vnd.kde.kformula kfo
application/vnd.kde.kivio flw
application/vnd.kde.kontour kon
application/vnd.kde.kpresenter kpr kpt
application/vnd.kde.kspread ksp
application/vnd.kde.kword kwd kwt
application/vnd.kenameaapp htke
application/vnd.kidspiration kia
application/vnd.kinar kne knp
application/vnd.koan skp skd skt skm
application/vnd.kodak-descriptor sse
application/vnd.las.las+xml lasxml
application/vnd.llamagraphics.life-balance.desktop lbd
application/vnd.llamagraphics.life-balance.exchange+xml lbe
application/vnd.lotus-1-2-3 123
application/vnd.lotus-approach apr
application/vnd.lotus-freelance pre
application/vnd.lotus-notes nsf
application/vnd.lotus-organizer org
application/vnd.lotus-screencam scm
application/vnd.lotus-wordpro lwp
application/vnd.macports.portpkg portpkg
application/vnd.mcd mcd
application/vnd.medcalcdata mc1
application/vnd.mediastation.cdkey cdkey
application/vnd.mfer mwf
application/vnd.mfmp mfm
application/vnd.micrografx.flo flo
application/vnd.micrografx.igx igx
application/vnd.mif mif
application/vnd.mobius.daf daf
application/vnd.mobius.dis dis
application/vnd.mobius.mbk mbk
application/vnd.mobius.mqy mqy
application/vnd.mobius.msl msl
application/vnd.mobius.plc plc
application/vnd.mobius.txf txf
application/vnd.mophun.application mpn
application/vnd.mophun.certificate mpc
application/vnd.mozilla.xul+xml xul
application/vnd.ms-artgalry cil
application/vnd.ms-cab-compressed cab
application/vnd.ms-excel xls xlm xla xlc xlt xlw
application/vnd.ms-excel.addin.macroenabled.12 xlam
application/vnd.ms-excel.sheet.binary.macroenabled.12 xlsb
application/vnd.ms-excel.sheet.macroenabled.12 xlsm
application/vnd.ms-excel.template.macroenabled.12 xltm
application/vnd.ms-fontobject eot
application/vnd.ms-htmlhelp chm
application/vnd.ms-ims ims
application/vnd.ms-lrm lrm
application/vnd.ms-officetheme thmx
application/vnd.ms-pki.seccat cat
application/vnd.ms-pki.stl stl
application/vnd.ms-powerpoint ppt pps pot
application/vnd.ms-powerpoint.addin.macroenabled.12 ppam
application/vnd.ms-powerpoint.presentation.macroenabled.12 pptm
application/vnd.ms-powerpoint.slide.macroenabled.12 sldm
application/vnd.ms-powerpoint.slideshow.macroenabled.12 ppsm
application/vnd.ms-powerpoint.template.macroenabled.12 potm
application/vnd.ms-project mpp mpt
application/vnd.ms-word.document.macroenabled.12 docm
application/vnd.ms-word.template.macroenabled.12 dotm
application/vnd.ms-works wps wks wcm wdb
application/vnd.ms-wpl wpl
application/vnd.ms-xpsdocument xps
application/vnd.mseq mseq
application/vnd.musician mus
application/vnd.muvee.style msty
application/vnd.mynfc taglet
application/vnd.neurolanguage.nlu nlu
application/vnd.nitf ntf nitf
application/vnd.noblenet-directory nnd
application/vnd.noblenet-sealer nns
application/vnd.noblenet-web nnw
application/vnd.nokia.n-gage.data ngdat
application/vnd.nokia.n-gage.symbian.install n-gage
application/vnd.nokia.radio-preset rpst
application/vnd.nokia.radio-presets rpss
application/vnd.novadigm.edm edm
application/vnd.novadigm.edx edx
application/vnd.novadigm.ext ext
application/vnd.oasis.opendocument.chart odc
application/vnd.oasis.opendocument.chart-template otc
application/vnd.oasis.opendocument.database odb
application/vnd.oasis.opendocument.formula odf
application/vnd.oasis.opendocument.formula-template odft
application/vnd.oasis.opendocument.graphics odg
application/vnd.oasis.opendocument.graphics-template otg
application/vnd.oasis.opendocument.image odi
application/vnd.oasis.opendocument.image-template oti
application/vnd.oasis.opendocument.presentation odp
application/vnd.oasis.opendocument.presentation-template otp
application/vnd.oasis.opendocument.spreadsheet ods
application/vnd.oasis.opendocument.spreadsheet-template ots
application/vnd.oasis.opendocument.text odt
application/vnd.oasis.opendocument.text-master odm
application/vnd.oasis.opendocument.text-template ott
application/vnd.oasis.opendocument.text-web oth
application/vnd.olpc-sugar xo
application/vnd.oma.dd2+xml dd2
application/vnd.openofficeorg.extension oxt
application/vnd.openxmlformats-officedocument.presentationml.presentation pptx
application/vnd.openxmlformats-officedocument.presentationml.slide sldx
application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx
application/vnd.openxmlformats-officedocument.presentationml.template potx
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx
application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx
application/vnd.openxmlformats-officedocument.wordprocessingml.document docx
application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx
application/vnd.osgeo.mapguide.package mgp
application/vnd.osgi.dp dp
application/vnd.osgi.subsystem esa
application/vnd.palm pdb pqa oprc
application/vnd.pawaafile paw
application/vnd.pg.format str
application/vnd.pg.osasli ei6
application/vnd.picsel efif
application/vnd.pmi.widget wg
application/vnd.pocketlearn plf
application/vnd.powerbuilder6 pbd
application/vnd.previewsystems.box box
application/vnd.proteus.magazine mgz
application/vnd.publishare-delta-tree qps
application/vnd.pvi.ptid1 ptid
application/vnd.quark.quarkxpress qxd qxt qwd qwt qxl qxb
application/vnd.realvnc.bed bed
application/vnd.recordare.musicxml mxl
application/vnd.recordare.musicxml+xml musicxml
application/vnd.rig.cryptonote cryptonote
application/vnd.rim.cod cod
application/vnd.rn-realmedia rm
application/vnd.rn-realmedia-vbr rmvb
application/vnd.route66.link66+xml link66
application/vnd.sailingtracker.track st
application/vnd.seemail see
application/vnd.sema sema
application/vnd.semd semd
application/vnd.semf semf
application/vnd.shana.informed.formdata ifm
application/vnd.shana.informed.formtemplate itp
application/vnd.shana.informed.interchange iif
application/vnd.shana.informed.package ipk
application/vnd.simtech-mindmapper twd twds
application/vnd.smaf mmf
application/vnd.smart.teacher teacher
application/vnd.solent.sdkm+xml sdkm sdkd
application/vnd.spotfire.dxp dxp
application/vnd.spotfire.sfs sfs
application/vnd.stardivision.calc sdc
application/vnd.stardivision.draw sda
application/vnd.stardivision.impress sdd
application/vnd.stardivision.math smf
application/vnd.stardivision.writer sdw vor
application/vnd.stardivision.writer-global sgl
application/vnd.stepmania.package smzip
application/vnd.stepmania.stepchart sm
application/vnd.sun.xml.calc sxc
application/vnd.sun.xml.calc.template stc
application/vnd.sun.xml.draw sxd
application/vnd.sun.xml.draw.template std
application/vnd.sun.xml.impress sxi
application/vnd.sun.xml.impress.template sti
application/vnd.sun.xml.math sxm
application/vnd.sun.xml.writer sxw
application/vnd.sun.xml.writer.global sxg
application/vnd.sun.xml.writer.template stw
application/vnd.sus-calendar sus susp
application/vnd.svd svd
application/vnd.symbian.install sis sisx
application/vnd.syncml+xml xsm
application/vnd.syncml.dm+wbxml bdm
application/vnd.syncml.dm+xml xdm
application/vnd.tao.intent-module-archive tao
application/vnd.tcpdump.pcap pcap cap dmp
application/vnd.tmobile-livetv tmo
application/vnd.trid.tpt tpt
application/vnd.triscape.mxs mxs
application/vnd.trueapp tra
application/vnd.ufdl ufd ufdl
application/vnd.uiq.theme utz
application/vnd.umajin umj
application/vnd.unity unityweb
application/vnd.uoml+xml uoml
application/vnd.vcx vcx
application/vnd.visio vsd vst vss vsw
application/vnd.visionary vis
application/vnd.vsf vsf
application/vnd.wap.wbxml wbxml
application/vnd.wap.wmlc wmlc
application/vnd.wap.wmlscriptc wmlsc
application/vnd.webturbo wtb
application/vnd.wolfram.player nbp
application/vnd.wordperfect wpd
application/vnd.wqd wqd
application/vnd.wt.stf stf
application/vnd.xara xar
application/vnd.xfdl xfdl
application/vnd.yamaha.hv-dic hvd
application/vnd.yamaha.hv-script hvs
application/vnd.yamaha.hv-voice hvp
application/vnd.yamaha.openscoreformat osf
application/vnd.yamaha.openscoreformat.osfpvg+xml osfpvg
application/vnd.yamaha.smaf-audio saf
application/vnd.yamaha.smaf-phrase spf
application/vnd.yellowriver-custom-menu cmp
application/vnd.zul zir zirz
application/vnd.zzazz.deck+xml zaz
application/voicexml+xml vxml
application/widget wgt
application/winhlp hlp
application/wsdl+xml wsdl
application/wspolicy+xml wspolicy
application/x-7z-compressed 7z
application/x-abiword abw
application/x-ace-compressed ace
application/x-apple-diskimage dmg
application/x-authorware-bin aab x32 u32 vox
application/x-authorware-map aam
application/x-authorware-seg aas
application/x-bcpio bcpio
application/x-bittorrent torrent
application/x-blorb blb blorb
application/x-bzip bz
application/x-bzip2 bz2 boz
application/x-cbr cbr cba cbt cbz cb7
application/x-cdlink vcd
application/x-cfs-compressed cfs
application/x-chat chat
application/x-chess-pgn pgn
application/x-conference nsc
application/x-cpio cpio
application/x-csh csh
application/x-debian-package deb udeb
application/x-dgc-compressed dgc
application/x-director dir dcr dxr cst cct cxt w3d fgd swa
application/x-doom wad
application/x-dtbncx+xml ncx
application/x-dtbook+xml dtb
application/x-dtbresource+xml res
application/x-dvi dvi
application/x-envoy evy
application/x-eva eva
application/x-fictionbook+xml fb2
application/x-font-bdf bdf
application/x-font-ghostscript gsf
application/x-font-linux-psf psf
application/x-font-otf otf
application/x-font-pcf pcf
application/x-font-snf snf
application/x-font-ttf ttf ttc
application/x-font-type1 pfa pfb pfm afm
application/x-font-woff woff
application/x-freearc arc
application/x-futuresplash spl
application/x-gca-compressed gca
application/x-glulx ulx
application/x-gnumeric gnumeric
application/x-gramps-xml gramps
application/x-gtar gtar
application/x-hdf hdf
application/x-install-instructions install
application/x-iso9660-image iso
application/x-java-jnlp-file jnlp
application/x-latex latex
application/x-lzh-compressed lzh lha
application/x-mie mie
application/x-mobipocket-ebook prc mobi
application/x-ms-application application
application/x-ms-shortcut lnk
application/x-ms-wmd wmd
application/x-ms-wmz wmz
application/x-ms-xbap xbap
application/x-msaccess mdb
application/x-msbinder obd
application/x-mscardfile crd
application/x-msclip clp
application/x-msdownload exe dll com bat msi
application/x-msmediaview mvb m13 m14
application/x-msmetafile wmf wmz emf emz
application/x-msmoney mny
application/x-mspublisher pub
application/x-msschedule scd
application/x-msterminal trm
application/x-mswrite wri
application/x-netcdf nc cdf
application/x-nzb nzb
application/x-pkcs12 p12 pfx
application/x-pkcs7-certificates p7b spc
application/x-pkcs7-certreqresp p7r
application/x-rar-compressed rar
application/x-research-info-systems ris
application/x-sh sh
application/x-shar shar
application/x-shockwave-flash swf
application/x-silverlight-app xap
application/x-sql sql
application/x-stuffit sit
application/x-stuffitx sitx
application/x-subrip srt
application/x-sv4cpio sv4cpio
application/x-sv4crc sv4crc
application/x-t3vm-image t3
application/x-tads gam
application/x-tar tar
application/x-tcl tcl
application/x-tex tex
application/x-tex-tfm tfm
application/x-texinfo texinfo texi
application/x-tgif obj
application/x-tgsticker tgs
application/x-tgwallpattern tgv
application/x-ustar ustar
application/x-wais-source src
application/x-x509-ca-cert der crt
application/x-xfig fig
application/x-xliff+xml xlf
application/x-xpinstall xpi
application/x-xz xz
application/x-zmachine z1 z2 z3 z4 z5 z6 z7 z8
application/xaml+xml xaml
application/xcap-diff+xml xdf
application/xenc+xml xenc
application/xhtml+xml xhtml xht
application/xml xml xsl
application/xml-dtd dtd
application/xop+xml xop
application/xproc+xml xpl
application/xslt+xml xslt
application/xspf+xml xspf
application/xv+xml mxml xhvml xvml xvm
application/yang yang
application/yin+xml yin
application/zip zip
audio/adpcm adp
audio/basic au snd
audio/midi mid midi kar rmi
audio/mp4 m4a mp4a
audio/mpeg mpga mp2 mp2a mp3 m2a m3a
audio/ogg oga ogg opus spx
audio/s3m s3m
audio/silk sil
audio/vnd.dece.audio uva uvva
audio/vnd.digital-winds eol
audio/vnd.dra dra
audio/vnd.dts dts
audio/vnd.dts.hd dtshd
audio/vnd.lucent.voice lvp
audio/vnd.ms-playready.media.pya pya
audio/vnd.nuera.ecelp4800 ecelp4800
audio/vnd.nuera.ecelp7470 ecelp7470
audio/vnd.nuera.ecelp9600 ecelp9600
audio/vnd.rip rip
audio/webm weba
audio/x-aac aac
audio/x-aiff aif aiff aifc
audio/x-caf caf
audio/x-flac flac
audio/x-matroska mka
audio/x-mpegurl m3u
audio/x-ms-wax wax
audio/x-ms-wma wma
audio/x-pn-realaudio ram ra
audio/x-pn-realaudio-plugin rmp
audio/x-wav wav
audio/xm xm
chemical/x-cdx cdx
chemical/x-cif cif
chemical/x-cmdf cmdf
chemical/x-cml cml
chemical/x-csml csml
chemical/x-xyz xyz
font/collection ttc
font/otf otf
font/ttf ttf
font/woff woff
font/woff2 woff2
image/bmp bmp
image/cgm cgm
image/g3fax g3
image/gif gif
image/heic heic
image/heic-sequence heics
image/heif heif
image/heif-sequence heifs
image/ief ief
image/jpeg jpeg jpg jpe
image/ktx ktx
image/png png
image/prs.btif btif
image/sgi sgi
image/svg+xml svg svgz
image/tiff tiff tif
image/vnd.adobe.photoshop psd
image/vnd.dece.graphic uvi uvvi uvg uvvg
image/vnd.djvu djvu djv
image/vnd.dvb.subtitle sub
image/vnd.dwg dwg
image/vnd.dxf dxf
image/vnd.fastbidsheet fbs
image/vnd.fpx fpx
image/vnd.fst fst
image/vnd.fujixerox.edmics-mmr mmr
image/vnd.fujixerox.edmics-rlc rlc
image/vnd.ms-modi mdi
image/vnd.ms-photo wdp
image/vnd.net-fpx npx
image/vnd.wap.wbmp wbmp
image/vnd.xiff xif
image/webp webp
image/x-3ds 3ds
image/x-cmu-raster ras
image/x-cmx cmx
image/x-freehand fh fhc fh4 fh5 fh7
image/x-icon ico
image/x-mrsid-image sid
image/x-pcx pcx
image/x-pict pic pct
image/x-portable-anymap pnm
image/x-portable-bitmap pbm
image/x-portable-graymap pgm
image/x-portable-pixmap ppm
image/x-rgb rgb
image/x-tga tga
image/x-xbitmap xbm
image/x-xpixmap xpm
image/x-xwindowdump xwd
message/rfc822 eml mime
model/iges igs iges
model/mesh msh mesh silo
model/vnd.collada+xml dae
model/vnd.dwf dwf
model/vnd.gdl gdl
model/vnd.gtw gtw
model/vnd.mts mts
model/vnd.vtu vtu
model/vrml wrl vrml
model/x3d+binary x3db x3dbz
model/x3d+vrml x3dv x3dvz
model/x3d+xml x3d x3dz
text/cache-manifest appcache
text/calendar ics ifb
text/css css
text/csv csv
text/html html htm
text/n3 n3
text/plain txt text conf def list log in
text/prs.lines.tag dsc
text/richtext rtx
text/sgml sgml sgm
text/tab-separated-values tsv
text/troff t tr roff man me ms
text/turtle ttl
text/uri-list uri uris urls
text/vcard vcard
text/vnd.curl curl
text/vnd.curl.dcurl dcurl
text/vnd.curl.mcurl mcurl
text/vnd.curl.scurl scurl
text/vnd.dvb.subtitle sub
text/vnd.fly fly
text/vnd.fmi.flexstor flx
text/vnd.graphviz gv
text/vnd.in3d.3dml 3dml
text/vnd.in3d.spot spot
text/vnd.sun.j2me.app-descriptor jad
text/vnd.wap.wml wml
text/vnd.wap.wmlscript wmls
text/x-asm s asm
text/x-c c cc cxx cpp h hh dic
text/x-fortran f for f77 f90
text/x-java-source java
text/x-nfo nfo
text/x-opml opml
text/x-pascal p pas
text/x-php php
text/x-setext etx
text/x-sfv sfv
text/x-uuencode uu
text/x-vcalendar vcs
text/x-vcard vcf
video/3gpp 3gp
video/3gpp2 3g2
video/h261 h261
video/h263 h263
video/h264 h264
video/h265 h265
video/jpeg jpgv
video/jpm jpm jpgm
video/mj2 mj2 mjp2
video/mp4 mp4 mp4v mpg4
video/mpeg mpeg mpg mpe m1v m2v
video/ogg ogv
video/quicktime qt mov
video/vnd.dece.hd uvh uvvh
video/vnd.dece.mobile uvm uvvm
video/vnd.dece.pd uvp uvvp
video/vnd.dece.sd uvs uvvs
video/vnd.dece.video uvv uvvv
video/vnd.dvb.file dvb
video/vnd.fvt fvt
video/vnd.mpegurl mxu m4u
video/vnd.ms-playready.media.pyv pyv
video/vnd.uvvu.mp4 uvu uvvu
video/vnd.vivo viv
video/webm webm
video/x-f4v f4v
video/x-fli fli
video/x-flv flv
video/x-m4v m4v
video/x-matroska mkv mk3d mks
video/x-mng mng
video/x-ms-asf asf asx
video/x-ms-vob vob
video/x-ms-wm wm
video/x-ms-wmv wmv
video/x-ms-wmx wmx
video/x-ms-wvx wvx
video/x-msvideo avi
video/x-sgi-movie movie
video/x-smv smv
x-conference/x-cooltalk ice

51
td/tdutils/td/utils/AesCtrByteFlow.h Normal file
View File

@@ -0,0 +1,51 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/ByteFlow.h"
#include "td/utils/common.h"
#include "td/utils/crypto.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/UInt.h"
namespace td {
#if TD_HAVE_OPENSSL
class AesCtrByteFlow final : public ByteFlowInplaceBase {
public:
void init(const UInt256 &key, const UInt128 &iv) {
state_.init(as_slice(key), as_slice(iv));
}
void init(AesCtrState &&state) {
state_ = std::move(state);
}
AesCtrState move_aes_ctr_state() {
return std::move(state_);
}
bool loop() final {
bool result = false;
auto ready = input_->prepare_read();
if (!ready.empty()) {
state_.encrypt(ready, MutableSlice(const_cast<char *>(ready.data()), ready.size()));
input_->confirm_read(ready.size());
output_.advance_end(ready.size());
result = true;
}
if (!is_input_active_) {
finish(Status::OK()); // End of input stream.
}
return result;
}
private:
AesCtrState state_;
};
#endif
} // namespace td

165
td/tdutils/td/utils/AsyncFileLog.cpp Normal file
View File

@@ -0,0 +1,165 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/AsyncFileLog.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/port/path.h"
#include "td/utils/port/sleep.h"
#include "td/utils/port/StdStreams.h"
#include "td/utils/SliceBuilder.h"
#include "td/utils/Time.h"
namespace td {
#if !TD_THREAD_UNSUPPORTED
Status AsyncFileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) {
CHECK(path_.empty());
CHECK(!path.empty());
TRY_RESULT(fd, FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append));
if (!Stderr().empty() && redirect_stderr) {
fd.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
auto r_path = realpath(path, true);
if (r_path.is_error()) {
path_ = std::move(path);
} else {
path_ = r_path.move_as_ok();
}
TRY_RESULT(size, fd.get_size());
queue_ = td::make_unique<MpscPollableQueue<Query>>();
queue_->init();
logging_thread_ = td::thread(
[queue = queue_.get(), fd = std::move(fd), path = path_, size, rotate_threshold, redirect_stderr]() mutable {
auto after_rotation = [&] {
fd.close();
auto r_fd = FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append);
if (r_fd.is_error()) {
process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__ << '\n');
}
fd = r_fd.move_as_ok();
if (!Stderr().empty() && redirect_stderr) {
fd.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
auto r_size = fd.get_size();
if (r_fd.is_error()) {
process_fatal_error(PSLICE() << "Failed to get log size: " << r_fd.error() << " in " << __FILE__ << " at "
<< __LINE__ << '\n');
}
size = r_size.move_as_ok();
};
auto append = [&](CSlice slice) {
if (size > rotate_threshold) {
auto status = rename(path, PSLICE() << path << ".old");
if (status.is_error()) {
process_fatal_error(PSLICE() << status << " in " << __FILE__ << " at " << __LINE__ << '\n');
}
after_rotation();
}
while (!slice.empty()) {
if (redirect_stderr) {
while (has_log_guard()) {
// spin
}
}
auto r_size = fd.write(slice);
if (r_size.is_error()) {
process_fatal_error(PSLICE() << r_size.error() << " in " << __FILE__ << " at " << __LINE__ << '\n');
}
auto written = r_size.ok();
size += static_cast<int64>(written);
slice.remove_prefix(written);
}
};
while (true) {
int ready_count = queue->reader_wait_nonblock();
if (ready_count == 0) {
queue->reader_get_event_fd().wait(1000);
continue;
}
bool need_close = false;
while (ready_count-- > 0) {
Query query = queue->reader_get_unsafe();
switch (query.type_) {
case Query::Type::Log:
append(query.data_);
break;
case Query::Type::AfterRotation:
after_rotation();
break;
case Query::Type::Close:
need_close = true;
break;
default:
process_fatal_error("Invalid query type in AsyncFileLog");
}
}
queue->reader_flush();
if (need_close) {
fd.close();
break;
}
}
});
return Status::OK();
}
AsyncFileLog::~AsyncFileLog() {
if (queue_ == nullptr) {
return;
}
Query query;
query.type_ = Query::Type::Close;
queue_->writer_put(std::move(query));
logging_thread_.join();
}
vector<string> AsyncFileLog::get_file_paths() {
vector<string> result;
if (!path_.empty()) {
result.push_back(path_);
result.push_back(PSTRING() << path_ << ".old");
}
return result;
}
void AsyncFileLog::after_rotation() {
Query query;
query.type_ = Query::Type::AfterRotation;
if (queue_ == nullptr) {
process_fatal_error("AsyncFileLog is not inited");
}
queue_->writer_put(std::move(query));
}
void AsyncFileLog::do_append(int log_level, CSlice slice) {
Query query;
query.data_ = slice.str();
if (queue_ == nullptr) {
process_fatal_error("AsyncFileLog is not inited");
}
queue_->writer_put(std::move(query));
if (log_level == VERBOSITY_NAME(FATAL)) {
// it is not thread-safe to join logging_thread_ there, so just wait for the log line to be printed
auto end_time = Time::now() + 1.0;
while (!queue_->is_empty() && Time::now() < end_time) {
usleep_for(1000);
}
usleep_for(5000); // allow some time for the log line to be actually printed
}
}
#endif
} // namespace td

51
td/tdutils/td/utils/AsyncFileLog.h Normal file
View File

@@ -0,0 +1,51 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/MpscPollableQueue.h"
#include "td/utils/port/thread.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
#if !TD_THREAD_UNSUPPORTED
class AsyncFileLog final : public LogInterface {
public:
AsyncFileLog() = default;
AsyncFileLog(const AsyncFileLog &) = delete;
AsyncFileLog &operator=(const AsyncFileLog &) = delete;
AsyncFileLog(AsyncFileLog &&) = delete;
AsyncFileLog &operator=(AsyncFileLog &&) = delete;
~AsyncFileLog();
Status init(string path, int64 rotate_threshold, bool redirect_stderr = true);
private:
struct Query {
enum class Type : int32 { Log, AfterRotation, Close };
Type type_ = Type::Log;
string data_;
};
string path_;
unique_ptr<MpscPollableQueue<Query>> queue_;
thread logging_thread_;
vector<string> get_file_paths() final;
void after_rotation() final;
void do_append(int log_level, CSlice slice) final;
};
#endif
} // namespace td

89
td/tdutils/td/utils/AtomicRead.h Normal file
View File

@@ -0,0 +1,89 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/sleep.h"
#include "td/utils/type_traits.h"
#include <atomic>
#include <cstring>
#include <memory>
namespace td {
template <class T>
class AtomicRead {
public:
void read(T &dest) const {
uint32 counter = 0;
auto wait = [&] {
counter++;
const int wait_each_count = 4;
if (counter % wait_each_count == 0) {
usleep_for(1);
}
};
while (true) {
static_assert(TD_IS_TRIVIALLY_COPYABLE(T), "T must be trivially copyable");
auto version_before = version.load();
if (version_before % 2 == 0) {
std::memcpy(&dest, &value, sizeof(dest));
auto version_after = version.load();
if (version_before == version_after) {
break;
}
}
wait();
}
}
struct Write {
explicit Write(AtomicRead *read) {
read->do_lock();
ptr.reset(read);
}
struct Destructor {
void operator()(AtomicRead *read) const {
read->do_unlock();
}
};
T &operator*() {
return value();
}
T *operator->() {
return &value();
}
T &value() {
CHECK(ptr);
return ptr->value;
}
private:
std::unique_ptr<AtomicRead, Destructor> ptr;
};
Write lock() {
return Write(this);
}
private:
std::atomic<uint64> version{0};
T value;
void do_lock() {
bool is_locked = ++version % 2 == 1;
CHECK(is_locked);
}
void do_unlock() {
bool is_unlocked = ++version % 2 == 0;
CHECK(is_unlocked);
}
};
} // namespace td

321
td/tdutils/td/utils/BigNum.cpp Normal file
View File

@@ -0,0 +1,321 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/BigNum.h"
char disable_linker_warning_about_empty_file_bignum_cpp TD_UNUSED;
#if TD_HAVE_OPENSSL
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/SliceBuilder.h"
#include <openssl/bn.h>
#include <openssl/crypto.h>
#include <algorithm>
namespace td {
class BigNumContext::Impl {
public:
BN_CTX *big_num_context;
Impl() : big_num_context(BN_CTX_new()) {
LOG_IF(FATAL, big_num_context == nullptr);
}
Impl(const Impl &) = delete;
Impl &operator=(const Impl &) = delete;
Impl(Impl &&) = delete;
Impl &operator=(Impl &&) = delete;
~Impl() {
BN_CTX_free(big_num_context);
}
};
BigNumContext::BigNumContext() : impl_(make_unique<Impl>()) {
}
BigNumContext::BigNumContext(BigNumContext &&) noexcept = default;
BigNumContext &BigNumContext::operator=(BigNumContext &&) noexcept = default;
BigNumContext::~BigNumContext() = default;
class BigNum::Impl {
public:
BIGNUM *big_num;
Impl() : Impl(BN_new()) {
}
explicit Impl(BIGNUM *big_num) : big_num(big_num) {
LOG_IF(FATAL, big_num == nullptr);
}
Impl(const Impl &) = delete;
Impl &operator=(const Impl &) = delete;
Impl(Impl &&) = delete;
Impl &operator=(Impl &&) = delete;
~Impl() {
BN_clear_free(big_num);
}
};
BigNum::BigNum() : impl_(make_unique<Impl>()) {
}
BigNum::BigNum(const BigNum &other) : BigNum() {
*this = other;
}
BigNum &BigNum::operator=(const BigNum &other) {
if (this == &other) {
return *this;
}
CHECK(impl_ != nullptr);
CHECK(other.impl_ != nullptr);
BIGNUM *result = BN_copy(impl_->big_num, other.impl_->big_num);
LOG_IF(FATAL, result == nullptr);
return *this;
}
BigNum::BigNum(BigNum &&) noexcept = default;
BigNum &BigNum::operator=(BigNum &&) noexcept = default;
BigNum::~BigNum() = default;
BigNum BigNum::from_binary(Slice str) {
return BigNum(make_unique<Impl>(BN_bin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
}
BigNum BigNum::from_le_binary(Slice str) {
#if defined(OPENSSL_IS_BORINGSSL)
return BigNum(make_unique<Impl>(BN_le2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
#elif OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
return BigNum(make_unique<Impl>(BN_lebin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
#else
string str_copy = str.str();
std::reverse(str_copy.begin(), str_copy.end());
return from_binary(str_copy);
#endif
}
Result<BigNum> BigNum::from_decimal(CSlice str) {
BigNum result;
int res = BN_dec2bn(&result.impl_->big_num, str.c_str());
if (res == 0 || static_cast<size_t>(res) != str.size()) {
return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as BigNum");
}
return result;
}
Result<BigNum> BigNum::from_hex(CSlice str) {
BigNum result;
int res = BN_hex2bn(&result.impl_->big_num, str.c_str());
if (res == 0 || static_cast<size_t>(res) != str.size()) {
return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as hexadecimal BigNum");
}
return result;
}
BigNum BigNum::from_raw(void *openssl_big_num) {
return BigNum(make_unique<Impl>(static_cast<BIGNUM *>(openssl_big_num)));
}
BigNum::BigNum(unique_ptr<Impl> &&impl) : impl_(std::move(impl)) {
}
int BigNum::get_num_bits() const {
return BN_num_bits(impl_->big_num);
}
int BigNum::get_num_bytes() const {
return BN_num_bytes(impl_->big_num);
}
void BigNum::set_bit(int num) {
int result = BN_set_bit(impl_->big_num, num);
LOG_IF(FATAL, result != 1);
}
void BigNum::clear_bit(int num) {
int result = BN_clear_bit(impl_->big_num, num);
LOG_IF(FATAL, result != 1);
}
bool BigNum::is_bit_set(int num) const {
return BN_is_bit_set(impl_->big_num, num) != 0;
}
bool BigNum::is_prime(BigNumContext &context) const {
#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER)
int result = BN_check_prime(impl_->big_num, context.impl_->big_num_context, nullptr);
#else
int result =
BN_is_prime_ex(impl_->big_num, get_num_bits() > 2048 ? 128 : 64, context.impl_->big_num_context, nullptr);
#endif
LOG_IF(FATAL, result == -1);
return result == 1;
}
void BigNum::operator+=(uint32 value) {
int result = BN_add_word(impl_->big_num, value);
LOG_IF(FATAL, result != 1);
}
void BigNum::operator-=(uint32 value) {
int result = BN_sub_word(impl_->big_num, value);
LOG_IF(FATAL, result != 1);
}
void BigNum::operator*=(uint32 value) {
int result = BN_mul_word(impl_->big_num, value);
LOG_IF(FATAL, result != 1);
}
void BigNum::operator/=(uint32 value) {
BN_ULONG result = BN_div_word(impl_->big_num, value);
LOG_IF(FATAL, result == static_cast<BN_ULONG>(-1));
}
uint32 BigNum::operator%(uint32 value) const {
BN_ULONG result = BN_mod_word(impl_->big_num, value);
LOG_IF(FATAL, result == static_cast<BN_ULONG>(-1));
return narrow_cast<uint32>(result);
}
void BigNum::set_value(uint32 new_value) {
if (new_value == 0) {
BN_zero(impl_->big_num);
} else {
int result = BN_set_word(impl_->big_num, new_value);
LOG_IF(FATAL, result != 1);
}
}
BigNum BigNum::clone() const {
BIGNUM *result = BN_dup(impl_->big_num);
LOG_IF(FATAL, result == nullptr);
return BigNum(make_unique<Impl>(result));
}
string BigNum::to_binary(int exact_size) const {
int num_size = get_num_bytes();
if (exact_size == -1) {
exact_size = num_size;
} else {
CHECK(exact_size >= num_size);
}
string res(exact_size, '\0');
BN_bn2bin(impl_->big_num, MutableSlice(res).ubegin() + (exact_size - num_size));
return res;
}
string BigNum::to_le_binary(int exact_size) const {
#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) || defined(OPENSSL_IS_BORINGSSL)
int num_size = get_num_bytes();
if (exact_size == -1) {
exact_size = num_size;
} else {
CHECK(exact_size >= num_size);
}
string result(exact_size, '\0');
#if defined(OPENSSL_IS_BORINGSSL)
BN_bn2le_padded(MutableSlice(result).ubegin(), exact_size, impl_->big_num);
#else
BN_bn2lebinpad(impl_->big_num, MutableSlice(result).ubegin(), exact_size);
#endif
return result;
#else
string result = to_binary(exact_size);
std::reverse(result.begin(), result.end());
return result;
#endif
}
string BigNum::to_decimal() const {
char *result = BN_bn2dec(impl_->big_num);
CHECK(result != nullptr);
string res(result);
OPENSSL_free(result);
return res;
}
void BigNum::random(BigNum &r, int bits, int top, int bottom) {
int result = BN_rand(r.impl_->big_num, bits, top, bottom);
LOG_IF(FATAL, result != 1);
}
void BigNum::add(BigNum &r, const BigNum &a, const BigNum &b) {
int result = BN_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num);
LOG_IF(FATAL, result != 1);
}
void BigNum::sub(BigNum &r, const BigNum &a, const BigNum &b) {
CHECK(r.impl_->big_num != a.impl_->big_num);
CHECK(r.impl_->big_num != b.impl_->big_num);
int result = BN_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num);
LOG_IF(FATAL, result != 1);
}
void BigNum::mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) {
int result = BN_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_add(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) {
int result = BN_mod_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_sub(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) {
int result = BN_mod_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) {
int result = BN_mod_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) {
auto result = BN_mod_inverse(r.impl_->big_num, a.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != r.impl_->big_num);
}
void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum &dividend, const BigNum &divisor,
BigNumContext &context) {
auto q = quotient == nullptr ? nullptr : quotient->impl_->big_num;
auto r = remainder == nullptr ? nullptr : remainder->impl_->big_num;
if (q == nullptr && r == nullptr) {
return;
}
auto result = BN_div(q, r, dividend.impl_->big_num, divisor.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context) {
int result = BN_mod_exp(r.impl_->big_num, a.impl_->big_num, p.impl_->big_num, m.impl_->big_num,
context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
void BigNum::gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) {
int result = BN_gcd(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context);
LOG_IF(FATAL, result != 1);
}
int BigNum::compare(const BigNum &a, const BigNum &b) {
return BN_cmp(a.impl_->big_num, b.impl_->big_num);
}
StringBuilder &operator<<(StringBuilder &sb, const BigNum &bn) {
return sb << bn.to_decimal();
}
} // namespace td
#endif

122
td/tdutils/td/utils/BigNum.h Normal file
View File

@@ -0,0 +1,122 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#if TD_HAVE_OPENSSL
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
namespace td {
class BigNumContext {
public:
BigNumContext();
BigNumContext(const BigNumContext &) = delete;
BigNumContext &operator=(const BigNumContext &) = delete;
BigNumContext(BigNumContext &&other) noexcept;
BigNumContext &operator=(BigNumContext &&other) noexcept;
~BigNumContext();
private:
class Impl;
unique_ptr<Impl> impl_;
friend class BigNum;
};
class BigNum {
public:
BigNum();
BigNum(const BigNum &other);
BigNum &operator=(const BigNum &other);
BigNum(BigNum &&other) noexcept;
BigNum &operator=(BigNum &&other) noexcept;
~BigNum();
static BigNum from_binary(Slice str);
static BigNum from_le_binary(Slice str);
static Result<BigNum> from_decimal(CSlice str);
static Result<BigNum> from_hex(CSlice str);
static BigNum from_raw(void *openssl_big_num);
void set_value(uint32 new_value);
int get_num_bits() const;
int get_num_bytes() const;
void set_bit(int num);
void clear_bit(int num);
bool is_bit_set(int num) const;
bool is_prime(BigNumContext &context) const;
BigNum clone() const;
string to_binary(int exact_size = -1) const;
string to_le_binary(int exact_size = -1) const;
string to_decimal() const;
void operator+=(uint32 value);
void operator-=(uint32 value);
void operator*=(uint32 value);
void operator/=(uint32 value);
uint32 operator%(uint32 value) const;
static void random(BigNum &r, int bits, int top, int bottom);
static void add(BigNum &r, const BigNum &a, const BigNum &b);
static void sub(BigNum &r, const BigNum &a, const BigNum &b);
static void mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context);
static void mod_add(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context);
static void mod_sub(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context);
static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context);
static void mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context);
static void div(BigNum *quotient, BigNum *remainder, const BigNum &dividend, const BigNum &divisor,
BigNumContext &context);
static void mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context);
static void gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context);
static int compare(const BigNum &a, const BigNum &b);
private:
class Impl;
unique_ptr<Impl> impl_;
explicit BigNum(unique_ptr<Impl> &&impl);
};
StringBuilder &operator<<(StringBuilder &sb, const BigNum &bn);
} // namespace td
#endif

231
td/tdutils/td/utils/BufferedFd.h Normal file
View File

@@ -0,0 +1,231 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include <limits>
namespace td {
// just reads from given reader and writes to given writer
template <class FdT>
class BufferedFdBase : public FdT {
public:
BufferedFdBase() = default;
explicit BufferedFdBase(FdT &&fd);
// TODO: make move constructor and move assignment safer
Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT;
Result<size_t> flush_write() TD_WARN_UNUSED_RESULT;
bool need_flush_write(size_t at_least = 0) {
return ready_for_flush_write() > at_least;
}
size_t ready_for_flush_write() {
CHECK(write_);
write_->sync_with_writer();
return write_->size();
}
void sync_with_poll() {
::td::sync_with_poll(*this);
}
void set_input_writer(ChainBufferWriter *read) {
read_ = read;
}
void set_output_reader(ChainBufferReader *write) {
write_ = write;
}
private:
ChainBufferWriter *read_ = nullptr;
ChainBufferReader *write_ = nullptr;
};
template <class FdT>
class BufferedFd final : public BufferedFdBase<FdT> {
using Parent = BufferedFdBase<FdT>;
ChainBufferWriter input_writer_;
ChainBufferReader input_reader_;
ChainBufferWriter output_writer_;
ChainBufferReader output_reader_;
void init();
void init_ptr();
public:
BufferedFd();
explicit BufferedFd(FdT &&fd);
BufferedFd(BufferedFd &&) noexcept;
BufferedFd &operator=(BufferedFd &&) noexcept;
BufferedFd(const BufferedFd &) = delete;
BufferedFd &operator=(const BufferedFd &) = delete;
~BufferedFd();
void close();
size_t left_unread() const {
return input_reader_.size();
}
size_t left_unwritten() const {
return output_reader_.size();
}
Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT;
Result<size_t> flush_write() TD_WARN_UNUSED_RESULT;
// Yep, direct access to buffers. It is IO interface too.
ChainBufferReader &input_buffer();
ChainBufferWriter &output_buffer();
};
// IMPLEMENTATION
/*** BufferedFd ***/
template <class FdT>
BufferedFdBase<FdT>::BufferedFdBase(FdT &&fd) : FdT(std::move(fd)) {
}
template <class FdT>
Result<size_t> BufferedFdBase<FdT>::flush_read(size_t max_read) {
CHECK(read_);
size_t result = 0;
while (::td::can_read_local(*this) && max_read) {
MutableSlice slice = read_->prepare_append();
slice.truncate(max_read);
TRY_RESULT(x, FdT::read(slice));
slice.truncate(x);
read_->confirm_append(x);
result += x;
max_read -= x;
}
return result;
}
template <class FdT>
Result<size_t> BufferedFdBase<FdT>::flush_write() {
// TODO: sync on demand
write_->sync_with_writer();
size_t result = 0;
while (!write_->empty() && ::td::can_write_local(*this)) {
constexpr size_t BUF_SIZE = 20;
IoSlice buf[BUF_SIZE];
auto it = write_->clone();
size_t buf_i;
for (buf_i = 0; buf_i < BUF_SIZE; buf_i++) {
Slice slice = it.prepare_read();
if (slice.empty()) {
break;
}
buf[buf_i] = as_io_slice(slice);
it.confirm_read(slice.size());
}
TRY_RESULT(x, FdT::writev(Span<IoSlice>(buf, buf_i)));
write_->advance(x);
result += x;
}
if (result == 0) {
if (write_->empty()) {
LOG(DEBUG) << "Nothing to write to " << FdT::get_poll_info().native_fd();
} else {
LOG(DEBUG) << "Can't flush write to " << FdT::get_poll_info().native_fd()
<< " with flags = " << FdT::get_poll_info().get_flags_local();
}
}
return result;
}
/*** BufferedFd ***/
template <class FdT>
void BufferedFd<FdT>::init() {
input_reader_ = input_writer_.extract_reader();
output_reader_ = output_writer_.extract_reader();
init_ptr();
}
template <class FdT>
void BufferedFd<FdT>::init_ptr() {
this->set_input_writer(&input_writer_);
this->set_output_reader(&output_reader_);
}
template <class FdT>
BufferedFd<FdT>::BufferedFd() {
init();
}
template <class FdT>
BufferedFd<FdT>::BufferedFd(FdT &&fd) : Parent(std::move(fd)) {
init();
}
template <class FdT>
BufferedFd<FdT>::BufferedFd(BufferedFd &&from) noexcept {
*this = std::move(from);
}
template <class FdT>
BufferedFd<FdT> &BufferedFd<FdT>::operator=(BufferedFd &&from) noexcept {
FdT::operator=(std::move(static_cast<FdT &>(from)));
input_reader_ = std::move(from.input_reader_);
input_writer_ = std::move(from.input_writer_);
output_reader_ = std::move(from.output_reader_);
output_writer_ = std::move(from.output_writer_);
init_ptr();
return *this;
}
template <class FdT>
BufferedFd<FdT>::~BufferedFd() {
close();
}
template <class FdT>
void BufferedFd<FdT>::close() {
FdT::close();
// TODO: clear buffers
}
template <class FdT>
Result<size_t> BufferedFd<FdT>::flush_read(size_t max_read) {
TRY_RESULT(result, Parent::flush_read(max_read));
if (result) {
// TODO: faster sync is possible if you owns writer.
input_reader_.sync_with_writer();
LOG(DEBUG) << "Flush read: +" << format::as_size(result) << tag("total", format::as_size(input_reader_.size()));
}
return result;
}
template <class FdT>
Result<size_t> BufferedFd<FdT>::flush_write() {
TRY_RESULT(result, Parent::flush_write());
if (result) {
LOG(DEBUG) << "Flush write: +" << format::as_size(result) << tag("left", format::as_size(output_reader_.size()));
}
return result;
}
// Yep, direct access to buffers. It is IO interface too.
template <class FdT>
ChainBufferReader &BufferedFd<FdT>::input_buffer() {
return input_reader_;
}
template <class FdT>
ChainBufferWriter &BufferedFd<FdT>::output_buffer() {
return output_writer_;
}
} // namespace td

61
td/tdutils/td/utils/BufferedReader.h Normal file
View File

@@ -0,0 +1,61 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
class BufferedReader {
public:
explicit BufferedReader(FileFd &file, size_t buff_size = 8152)
: file_(file), buff_(buff_size), begin_pos_(0), end_pos_(0) {
}
Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT;
private:
FileFd &file_;
vector<char> buff_;
size_t begin_pos_;
size_t end_pos_;
};
inline Result<size_t> BufferedReader::read(MutableSlice slice) {
size_t available = end_pos_ - begin_pos_;
if (available >= slice.size()) {
// have enough data in buffer
slice.copy_from({&buff_[begin_pos_], slice.size()});
begin_pos_ += slice.size();
return slice.size();
}
if (available) {
slice.copy_from({&buff_[begin_pos_], available});
begin_pos_ += available;
slice.remove_prefix(available);
}
if (slice.size() > buff_.size() / 2) {
TRY_RESULT(result, file_.read(slice));
return result + available;
}
TRY_RESULT(result, file_.read({&buff_[0], buff_.size()}));
begin_pos_ = 0;
end_pos_ = result;
size_t left = min(end_pos_, slice.size());
slice.copy_from({&buff_[0], left});
begin_pos_ = left;
return left + available;
}
} // namespace td

17
td/tdutils/td/utils/BufferedUdp.cpp Normal file
View File

@@ -0,0 +1,17 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/BufferedUdp.h"
char disable_linker_warning_about_empty_file_buffered_udp_cpp TD_UNUSED;
namespace td {
#if TD_PORT_POSIX
TD_THREAD_LOCAL detail::UdpReader *BufferedUdp::udp_reader_;
#endif
} // namespace td

177
td/tdutils/td/utils/BufferedUdp.h Normal file
View File

@@ -0,0 +1,177 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/optional.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/port/UdpSocketFd.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include "td/utils/VectorQueue.h"
#include <array>
namespace td {
#if TD_PORT_POSIX
namespace detail {
class UdpWriter {
public:
static Status write_once(UdpSocketFd &fd, VectorQueue<UdpMessage> &queue) TD_WARN_UNUSED_RESULT {
std::array<UdpSocketFd::OutboundMessage, 16> messages;
auto to_send = queue.as_span();
size_t to_send_n = td::min(messages.size(), to_send.size());
to_send.truncate(to_send_n);
for (size_t i = 0; i < to_send_n; i++) {
messages[i].to = &to_send[i].address;
messages[i].data = to_send[i].data.as_slice();
}
size_t cnt;
auto status = fd.send_messages(Span<UdpSocketFd::OutboundMessage>(messages).truncate(to_send_n), cnt);
queue.pop_n(cnt);
return status;
}
};
class UdpReaderHelper {
public:
void init_inbound_message(UdpSocketFd::InboundMessage &message) {
message.from = &message_.address;
message.error = &message_.error;
if (buffer_.size() < MAX_PACKET_SIZE) {
buffer_ = BufferSlice(RESERVED_SIZE);
}
CHECK(buffer_.size() >= MAX_PACKET_SIZE);
message.data = buffer_.as_mutable_slice().substr(0, MAX_PACKET_SIZE);
}
UdpMessage extract_udp_message(UdpSocketFd::InboundMessage &message) {
message_.data = buffer_.from_slice(message.data);
auto size = message_.data.size();
size = (size + 7) & ~7;
CHECK(size <= MAX_PACKET_SIZE);
buffer_.confirm_read(size);
return std::move(message_);
}
private:
static constexpr size_t MAX_PACKET_SIZE = 2048;
static constexpr size_t RESERVED_SIZE = MAX_PACKET_SIZE * 8;
UdpMessage message_;
BufferSlice buffer_;
};
// One for thread is enough
class UdpReader {
public:
UdpReader() {
for (size_t i = 0; i < messages_.size(); i++) {
helpers_[i].init_inbound_message(messages_[i]);
}
}
Status read_once(UdpSocketFd &fd, VectorQueue<UdpMessage> &queue) TD_WARN_UNUSED_RESULT {
for (auto &message : messages_) {
CHECK(message.data.size() == 2048);
}
size_t cnt = 0;
auto status = fd.receive_messages(messages_, cnt);
for (size_t i = 0; i < cnt; i++) {
queue.push(helpers_[i].extract_udp_message(messages_[i]));
helpers_[i].init_inbound_message(messages_[i]);
}
for (size_t i = cnt; i < messages_.size(); i++) {
LOG_CHECK(messages_[i].data.size() == 2048)
<< " cnt = " << cnt << " i = " << i << " size = " << messages_[i].data.size() << " status = " << status;
}
if (status.is_error() && !UdpSocketFd::is_critical_read_error(status)) {
queue.push(UdpMessage{{}, {}, std::move(status)});
status = Status::OK();
}
return status;
}
private:
static constexpr size_t BUFFER_SIZE = 16;
std::array<UdpSocketFd::InboundMessage, BUFFER_SIZE> messages_;
std::array<UdpReaderHelper, BUFFER_SIZE> helpers_;
};
} // namespace detail
#endif
class BufferedUdp final : public UdpSocketFd {
public:
explicit BufferedUdp(UdpSocketFd fd) : UdpSocketFd(std::move(fd)) {
}
#if TD_PORT_POSIX
void sync_with_poll() {
::td::sync_with_poll(*this);
}
Result<optional<UdpMessage>> receive() {
if (input_.empty() && can_read_local(*this)) {
TRY_STATUS(flush_read_once());
}
if (input_.empty()) {
return optional<UdpMessage>();
}
return input_.pop();
}
void send(UdpMessage message) {
output_.push(std::move(message));
}
Status flush_send() {
Status status;
while (status.is_ok() && can_write_local(*this) && !output_.empty()) {
status = flush_send_once();
}
return status;
}
#endif
UdpSocketFd move_as_udp_socket_fd() {
return std::move(as_fd());
}
UdpSocketFd &as_fd() {
return *static_cast<UdpSocketFd *>(this);
}
private:
#if TD_PORT_POSIX
VectorQueue<UdpMessage> input_;
VectorQueue<UdpMessage> output_;
VectorQueue<UdpMessage> &input() {
return input_;
}
VectorQueue<UdpMessage> &output() {
return output_;
}
Status flush_send_once() TD_WARN_UNUSED_RESULT {
return detail::UdpWriter::write_once(as_fd(), output_);
}
Status flush_read_once() TD_WARN_UNUSED_RESULT {
init_thread_local<detail::UdpReader>(udp_reader_);
return udp_reader_->read_once(as_fd(), input_);
}
static TD_THREAD_LOCAL detail::UdpReader *udp_reader_;
#endif
};
} // namespace td

389
td/tdutils/td/utils/ByteFlow.h Normal file
View File

@@ -0,0 +1,389 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/Status.h"
#include <limits>
namespace td {
class ByteFlowInterface {
public:
virtual void close_input(Status status) = 0;
virtual void wakeup() = 0;
virtual void set_parent(ByteFlowInterface &other) = 0;
virtual void set_input(ChainBufferReader *input) = 0;
virtual size_t get_need_size() = 0;
virtual size_t get_read_size() = 0;
virtual size_t get_write_size() = 0;
virtual void reset_need_size() {
}
ByteFlowInterface() = default;
ByteFlowInterface(const ByteFlowInterface &) = delete;
ByteFlowInterface &operator=(const ByteFlowInterface &) = delete;
ByteFlowInterface(ByteFlowInterface &&) = default;
ByteFlowInterface &operator=(ByteFlowInterface &&) = default;
virtual ~ByteFlowInterface() = default;
};
class ByteFlowBaseCommon : public ByteFlowInterface {
public:
ByteFlowBaseCommon() = default;
void close_input(Status status) final {
if (status.is_error()) {
finish(std::move(status));
} else {
is_input_active_ = false;
wakeup();
}
}
void wakeup() final {
if (stop_flag_ || !input_) {
return;
}
input_->sync_with_writer();
if (waiting_flag_) {
if (!is_input_active_) {
finish(Status::OK());
}
return;
}
while (true) {
if (stop_flag_) {
break;
}
// update can_read
if (is_input_active_) {
auto read_size = get_read_size();
if (read_size < min(need_size_, options_.read_watermark.low)) {
can_read = false;
}
if (read_size >= max(need_size_, options_.read_watermark.high)) {
can_read = true;
}
} else {
// always can read when input is closed
can_read = true;
}
// update can_write
{
auto write_size = get_write_size();
if (write_size > options_.write_watermark.high) {
can_write = false;
}
if (write_size <= options_.write_watermark.low) {
can_write = true;
}
}
if (!can_read || !can_write) {
break;
}
need_size_ = 0;
if (!loop()) {
if (need_size_ <= get_read_size()) {
need_size_ = get_read_size() + 1;
}
}
}
on_output_updated();
}
size_t get_need_size() final {
return need_size_;
}
void reset_need_size() override {
need_size_ = 0;
}
size_t get_read_size() override {
input_->sync_with_writer();
return input_->size();
}
size_t get_write_size() override {
CHECK(parent_);
return parent_->get_read_size();
}
struct Watermark {
size_t low{std::numeric_limits<size_t>::max()};
size_t high{0};
};
struct Options {
Watermark write_watermark;
Watermark read_watermark;
};
void set_options(Options options) {
options_ = options;
}
virtual bool loop() = 0;
protected:
bool waiting_flag_ = false;
ChainBufferReader *input_ = nullptr;
bool is_input_active_ = true;
size_t need_size_ = 0;
bool can_read{true};
bool can_write{true};
Options options_;
void finish(Status status) {
stop_flag_ = true;
need_size_ = 0;
if (parent_) {
parent_->close_input(std::move(status));
parent_ = nullptr;
}
}
void set_need_size(size_t need_size) {
need_size_ = need_size;
}
void on_output_updated() {
if (parent_) {
parent_->wakeup();
}
}
void consume_input() {
waiting_flag_ = true;
if (!is_input_active_) {
finish(Status::OK());
}
}
private:
ByteFlowInterface *parent_ = nullptr;
bool stop_flag_ = false;
friend class ByteFlowBase;
friend class ByteFlowInplaceBase;
};
class ByteFlowBase : public ByteFlowBaseCommon {
public:
ByteFlowBase() = default;
void set_input(ChainBufferReader *input) final {
input_ = input;
}
void set_parent(ByteFlowInterface &other) final {
parent_ = &other;
parent_->set_input(&output_reader_);
}
bool loop() override = 0;
// ChainBufferWriter &get_output() {
// return output_;
//}
protected:
ChainBufferWriter output_;
ChainBufferReader output_reader_ = output_.extract_reader();
};
class ByteFlowInplaceBase : public ByteFlowBaseCommon {
public:
ByteFlowInplaceBase() = default;
void set_input(ChainBufferReader *input) final {
input_ = input;
output_ = ChainBufferReader(input_->begin().clone(), input_->begin().clone(), false);
}
void set_parent(ByteFlowInterface &other) final {
parent_ = &other;
parent_->set_input(&output_);
}
bool loop() override = 0;
ChainBufferReader &get_output() {
return output_;
}
protected:
ChainBufferReader output_;
};
inline ByteFlowInterface &operator>>(ByteFlowInterface &from, ByteFlowInterface &to) {
from.set_parent(to);
return to;
}
class ByteFlowSource final : public ByteFlowInterface {
public:
ByteFlowSource() = default;
explicit ByteFlowSource(ChainBufferReader *buffer) : buffer_(buffer) {
}
ByteFlowSource(ByteFlowSource &&other) noexcept : buffer_(other.buffer_), parent_(other.parent_) {
other.buffer_ = nullptr;
other.parent_ = nullptr;
}
ByteFlowSource &operator=(ByteFlowSource &&other) noexcept {
buffer_ = other.buffer_;
parent_ = other.parent_;
other.buffer_ = nullptr;
other.parent_ = nullptr;
return *this;
}
ByteFlowSource(const ByteFlowSource &) = delete;
ByteFlowSource &operator=(const ByteFlowSource &) = delete;
~ByteFlowSource() override = default;
void set_input(ChainBufferReader *) final {
UNREACHABLE();
}
void set_parent(ByteFlowInterface &parent) final {
CHECK(parent_ == nullptr);
parent_ = &parent;
parent_->set_input(buffer_);
}
void close_input(Status status) final {
CHECK(parent_);
parent_->close_input(std::move(status));
parent_ = nullptr;
}
void wakeup() final {
if (!parent_) {
return;
}
parent_->wakeup();
}
size_t get_need_size() final {
if (parent_ == nullptr) {
return 0;
}
return parent_->get_need_size();
}
size_t get_read_size() final {
UNREACHABLE();
return 0;
}
size_t get_write_size() final {
CHECK(parent_);
return parent_->get_read_size();
}
private:
ChainBufferReader *buffer_ = nullptr;
ByteFlowInterface *parent_ = nullptr;
};
class ByteFlowSink final : public ByteFlowInterface {
public:
void set_input(ChainBufferReader *input) final {
CHECK(buffer_ == nullptr);
buffer_ = input;
}
void set_parent(ByteFlowInterface & /*parent*/) final {
UNREACHABLE();
}
void close_input(Status status) final {
CHECK(active_);
active_ = false;
status_ = std::move(status);
buffer_->sync_with_writer();
}
void wakeup() final {
buffer_->sync_with_writer();
}
size_t get_need_size() final {
UNREACHABLE();
return 0;
}
size_t get_read_size() final {
buffer_->sync_with_writer();
return buffer_->size();
}
size_t get_write_size() final {
UNREACHABLE();
return 0;
}
bool is_ready() {
return !active_;
}
Status &status() {
return status_;
}
ChainBufferReader *result() {
CHECK(is_ready() && status().is_ok());
return buffer_;
}
ChainBufferReader *get_output() {
return buffer_;
}
private:
bool active_ = true;
Status status_;
ChainBufferReader *buffer_ = nullptr;
};
class ByteFlowMoveSink final : public ByteFlowInterface {
public:
ByteFlowMoveSink() = default;
explicit ByteFlowMoveSink(ChainBufferWriter *output) {
set_output(output);
}
void set_input(ChainBufferReader *input) final {
CHECK(!input_);
input_ = input;
}
void set_parent(ByteFlowInterface & /*parent*/) final {
UNREACHABLE();
}
void close_input(Status status) final {
CHECK(active_);
active_ = false;
status_ = std::move(status);
wakeup();
}
void wakeup() final {
input_->sync_with_writer();
output_->append(*input_);
}
size_t get_need_size() final {
UNREACHABLE();
return 0;
}
size_t get_read_size() final {
input_->sync_with_writer();
//TODO: must be input_->size() + output_->size()
return input_->size();
}
size_t get_write_size() final {
UNREACHABLE();
return 0;
}
void set_output(ChainBufferWriter *output) {
CHECK(!output_);
output_ = output;
}
bool is_ready() {
return !active_;
}
Status &status() {
return status_;
}
private:
bool active_ = true;
Status status_;
ChainBufferReader *input_ = nullptr;
ChainBufferWriter *output_ = nullptr;
};
} // namespace td

71
td/tdutils/td/utils/CancellationToken.h Normal file
View File

@@ -0,0 +1,71 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include <atomic>
#include <memory>
namespace td {
namespace detail {
struct RawCancellationToken {
std::atomic<bool> is_canceled_{false};
};
} // namespace detail
class CancellationToken {
public:
explicit operator bool() const noexcept {
// empty CancellationToken is never canceled
if (!token_) {
return false;
}
return token_->is_canceled_.load(std::memory_order_acquire);
}
CancellationToken() = default;
explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
}
private:
std::shared_ptr<detail::RawCancellationToken> token_;
};
class CancellationTokenSource {
public:
CancellationTokenSource() = default;
CancellationTokenSource(CancellationTokenSource &&other) noexcept : token_(std::move(other.token_)) {
}
CancellationTokenSource &operator=(CancellationTokenSource &&other) noexcept {
cancel();
token_ = std::move(other.token_);
return *this;
}
CancellationTokenSource(const CancellationTokenSource &) = delete;
CancellationTokenSource &operator=(const CancellationTokenSource &) = delete;
~CancellationTokenSource() {
cancel();
}
CancellationToken get_cancellation_token() {
if (!token_) {
token_ = std::make_shared<detail::RawCancellationToken>();
}
return CancellationToken(token_);
}
void cancel() {
if (!token_) {
return;
}
token_->is_canceled_.store(true, std::memory_order_release);
token_.reset();
}
private:
std::shared_ptr<detail::RawCancellationToken> token_;
};
} // namespace td

385
td/tdutils/td/utils/ChainScheduler.h Normal file
View File

@@ -0,0 +1,385 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/algorithm.h"
#include "td/utils/common.h"
#include "td/utils/Container.h"
#include "td/utils/FlatHashMap.h"
#include "td/utils/FlatHashSet.h"
#include "td/utils/List.h"
#include "td/utils/logging.h"
#include "td/utils/optional.h"
#include "td/utils/Span.h"
#include "td/utils/StringBuilder.h"
#include "td/utils/VectorQueue.h"
#include <functional>
namespace td {
struct ChainSchedulerBase {
struct TaskWithParents {
uint64 task_id{};
vector<uint64> parents;
};
};
template <class ExtraT = Unit>
class ChainScheduler final : public ChainSchedulerBase {
public:
using TaskId = uint64;
using ChainId = uint64;
TaskId create_task(Span<ChainId> chains, ExtraT extra = {});
ExtraT *get_task_extra(TaskId task_id);
optional<TaskWithParents> start_next_task();
void pause_task(TaskId task_id);
void finish_task(TaskId task_id);
void reset_task(TaskId task_id);
template <class F>
void for_each(F &&f) {
tasks_.for_each([&f](uint64, Task &task) { f(task.extra); });
}
template <class F>
void for_each_dependent(TaskId task_id, F &&f) {
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
FlatHashSet<TaskId> visited;
bool check_for_collisions = task->chains.size() > 1;
for (TaskChainInfo &task_chain_info : task->chains) {
ChainInfo &chain_info = *task_chain_info.chain_info;
chain_info.chain.foreach_child(&task_chain_info.chain_node, [&](TaskId task_id, uint64) {
if (check_for_collisions && !visited.insert(task_id).second) {
return;
}
f(task_id);
});
}
}
private:
struct ChainNode : ListNode {
TaskId task_id{};
uint64 generation{};
};
class Chain {
public:
void add_task(ChainNode *node) {
head_.put_back(node);
}
optional<TaskId> get_first() {
if (head_.empty()) {
return {};
}
return static_cast<ChainNode &>(*head_.get_next()).task_id;
}
optional<TaskId> get_child(ChainNode *chain_node) {
if (chain_node->get_next() == head_.end()) {
return {};
}
return static_cast<ChainNode &>(*chain_node->get_next()).task_id;
}
optional<ChainNode *> get_parent(ChainNode *chain_node) {
if (chain_node->get_prev() == head_.end()) {
return {};
}
return static_cast<ChainNode *>(chain_node->get_prev());
}
void finish_task(ChainNode *node) {
node->remove();
}
bool empty() const {
return head_.empty();
}
void foreach(std::function<void(TaskId, uint64)> f) const {
for (auto it = head_.begin(); it != head_.end(); it = it->get_next()) {
auto &node = static_cast<const ChainNode &>(*it);
f(node.task_id, node.generation);
}
}
void foreach_child(ListNode *start_node, std::function<void(TaskId, uint64)> f) const {
for (auto it = start_node; it != head_.end(); it = it->get_next()) {
auto &node = static_cast<const ChainNode &>(*it);
f(node.task_id, node.generation);
}
}
private:
ListNode head_;
};
struct ChainInfo {
Chain chain;
uint32 active_tasks{};
uint64 generation{1};
};
struct TaskChainInfo {
ChainNode chain_node;
ChainId chain_id{};
ChainInfo *chain_info{};
};
struct Task {
enum class State { Pending, Active, Paused } state{State::Pending};
vector<TaskChainInfo> chains;
ExtraT extra;
};
FlatHashMap<ChainId, unique_ptr<ChainInfo>> chains_;
FlatHashMap<ChainId, TaskId> limited_tasks_;
Container<Task> tasks_;
VectorQueue<TaskId> pending_tasks_;
ChainInfo &get_chain_info(ChainId chain_id) {
auto &chain = chains_[chain_id];
if (chain == nullptr) {
chain = make_unique<ChainInfo>();
}
return *chain;
}
void try_start_task(TaskId task_id) {
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
if (task->state != Task::State::Pending) {
return;
}
for (TaskChainInfo &task_chain_info : task->chains) {
auto o_parent = task_chain_info.chain_info->chain.get_parent(&task_chain_info.chain_node);
if (o_parent) {
if (o_parent.value()->generation != task_chain_info.chain_info->generation) {
return;
}
}
if (task_chain_info.chain_info->active_tasks >= 10) {
limited_tasks_[task_chain_info.chain_id] = task_id;
return;
}
}
do_start_task(task_id, task);
}
void do_start_task(TaskId task_id, Task *task) {
for (TaskChainInfo &task_chain_info : task->chains) {
ChainInfo &chain_info = get_chain_info(task_chain_info.chain_id);
chain_info.active_tasks++;
task_chain_info.chain_node.generation = chain_info.generation;
}
task->state = Task::State::Active;
pending_tasks_.push(task_id);
for_each_child(task, [&](TaskId task_id) { try_start_task(task_id); });
}
template <class F>
void for_each_child(Task *task, F &&f) {
for (TaskChainInfo &task_chain_info : task->chains) {
ChainInfo &chain_info = *task_chain_info.chain_info;
auto o_child = chain_info.chain.get_child(&task_chain_info.chain_node);
if (o_child) {
f(o_child.value());
}
}
}
void inactivate_task(TaskId task_id, bool failed) {
LOG(DEBUG) << "Inactivate " << task_id << " " << (failed ? "failed" : "finished");
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
bool was_active = task->state == Task::State::Active;
task->state = Task::State::Pending;
for (TaskChainInfo &task_chain_info : task->chains) {
ChainInfo &chain_info = *task_chain_info.chain_info;
if (was_active) {
chain_info.active_tasks--;
}
if (was_active && failed) {
chain_info.generation = td::max(chain_info.generation, task_chain_info.chain_node.generation + 1);
}
auto it = limited_tasks_.find(task_chain_info.chain_id);
if (it != limited_tasks_.end()) {
auto limited_task_id = it->second;
limited_tasks_.erase(it);
if (limited_task_id != task_id) {
try_start_task_later(limited_task_id);
}
}
auto o_first = chain_info.chain.get_first();
if (o_first) {
auto first_task_id = o_first.unwrap();
if (first_task_id != task_id) {
try_start_task_later(first_task_id);
}
}
}
}
void finish_chain_task(TaskChainInfo &task_chain_info) {
auto &chain = task_chain_info.chain_info->chain;
chain.finish_task(&task_chain_info.chain_node);
if (chain.empty()) {
chains_.erase(task_chain_info.chain_id);
}
}
vector<TaskId> to_start_;
void try_start_task_later(TaskId task_id) {
LOG(DEBUG) << "Start later " << task_id;
to_start_.push_back(task_id);
}
void flush_try_start_task() {
auto moved_to_start = std::move(to_start_);
for (auto task_id : moved_to_start) {
try_start_task(task_id);
}
CHECK(to_start_.empty());
}
template <class ExtraTT>
friend StringBuilder &operator<<(StringBuilder &sb, ChainScheduler<ExtraTT> &scheduler);
};
template <class ExtraT>
typename ChainScheduler<ExtraT>::TaskId ChainScheduler<ExtraT>::create_task(Span<ChainId> chains, ExtraT extra) {
auto task_id = tasks_.create();
Task &task = *tasks_.get(task_id);
task.extra = std::move(extra);
task.chains = transform(chains, [&](ChainId chain_id) {
CHECK(chain_id != 0);
TaskChainInfo task_chain_info;
ChainInfo &chain_info = get_chain_info(chain_id);
task_chain_info.chain_id = chain_id;
task_chain_info.chain_info = &chain_info;
task_chain_info.chain_node.task_id = task_id;
task_chain_info.chain_node.generation = 0;
return task_chain_info;
});
for (TaskChainInfo &task_chain_info : task.chains) {
ChainInfo &chain_info = *task_chain_info.chain_info;
chain_info.chain.add_task(&task_chain_info.chain_node);
}
try_start_task(task_id);
return task_id;
}
// TODO: return reference
template <class ExtraT>
ExtraT *ChainScheduler<ExtraT>::get_task_extra(TaskId task_id) { // may return nullptr
auto *task = tasks_.get(task_id);
if (task == nullptr) {
return nullptr;
}
return &task->extra;
}
template <class ExtraT>
optional<ChainSchedulerBase::TaskWithParents> ChainScheduler<ExtraT>::start_next_task() {
if (pending_tasks_.empty()) {
return {};
}
auto task_id = pending_tasks_.pop();
TaskWithParents res;
res.task_id = task_id;
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
for (TaskChainInfo &task_chain_info : task->chains) {
Chain &chain = task_chain_info.chain_info->chain;
auto o_parent = chain.get_parent(&task_chain_info.chain_node);
if (o_parent) {
res.parents.push_back(o_parent.value()->task_id);
}
}
return res;
}
template <class ExtraT>
void ChainScheduler<ExtraT>::finish_task(TaskId task_id) {
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
CHECK(to_start_.empty());
inactivate_task(task_id, false);
for_each_child(task, [&](TaskId task_id) { try_start_task_later(task_id); });
for (TaskChainInfo &task_chain_info : task->chains) {
finish_chain_task(task_chain_info);
}
tasks_.erase(task_id);
flush_try_start_task();
}
template <class ExtraT>
void ChainScheduler<ExtraT>::reset_task(TaskId task_id) {
CHECK(to_start_.empty());
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
inactivate_task(task_id, true);
try_start_task_later(task_id);
flush_try_start_task();
}
template <class ExtraT>
void ChainScheduler<ExtraT>::pause_task(TaskId task_id) {
auto *task = tasks_.get(task_id);
CHECK(task != nullptr);
inactivate_task(task_id, true);
task->state = Task::State::Paused;
flush_try_start_task();
}
template <class ExtraT>
StringBuilder &operator<<(StringBuilder &sb, ChainScheduler<ExtraT> &scheduler) {
// 1 print chains
sb << '\n';
for (auto &it : scheduler.chains_) {
CHECK(it.second != nullptr);
sb << "ChainId{" << it.first << "}";
sb << " active_cnt = " << it.second->active_tasks;
sb << " g = " << it.second->generation;
sb << ':';
it.second->chain.foreach([&](typename ChainScheduler<ExtraT>::TaskId task_id, uint64 generation) {
sb << ' ' << *scheduler.get_task_extra(task_id) << ':' << generation;
});
sb << '\n';
}
scheduler.tasks_.for_each([&](uint64, typename ChainScheduler<ExtraT>::Task &task) {
sb << "Task: " << task.extra;
sb << " state = " << static_cast<int>(task.state);
for (auto &task_chain_info : task.chains) {
sb << " g = " << task_chain_info.chain_node.generation;
if (task_chain_info.chain_info->generation != task_chain_info.chain_node.generation) {
sb << " chain_g = " << task_chain_info.chain_info->generation;
}
}
sb << '\n';
});
return sb;
}
} // namespace td

61
td/tdutils/td/utils/ChangesProcessor.h Normal file
View File

@@ -0,0 +1,61 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <utility>
namespace td {
// Process changes after they are finished in order of addition
template <class DataT>
class ChangesProcessor {
public:
using Id = uint64;
void clear() {
offset_ += data_array_.size();
ready_i_ = 0;
data_array_.clear();
}
template <class FromDataT>
Id add(FromDataT &&data) {
auto res = offset_ + data_array_.size();
data_array_.emplace_back(std::forward<DataT>(data), false);
return static_cast<Id>(res);
}
template <class F>
void finish(Id token, F &&func) {
size_t pos = static_cast<size_t>(token) - offset_;
if (pos >= data_array_.size()) {
return;
}
data_array_[pos].second = true;
while (ready_i_ < data_array_.size() && data_array_[ready_i_].second == true) {
func(std::move(data_array_[ready_i_].first));
ready_i_++;
}
try_compactify();
}
private:
size_t offset_ = 1;
size_t ready_i_ = 0;
std::vector<std::pair<DataT, bool>> data_array_;
void try_compactify() {
if (ready_i_ > 5 && ready_i_ * 2 > data_array_.size()) {
data_array_.erase(data_array_.begin(), data_array_.begin() + ready_i_);
offset_ += ready_i_;
ready_i_ = 0;
}
}
};
} // namespace td

119
td/tdutils/td/utils/Closure.h Normal file
View File

@@ -0,0 +1,119 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/invoke.h"
#include <tuple>
#include <type_traits>
#include <utility>
//
// Essentially we have:
// (ActorT::func, arg1, arg2, ..., argn)
// We want to call:
// actor->func(arg1, arg2, ..., argn)
// And in some cases we would like to delay this call.
//
// First attempt would be
// [a1=arg1, a2=arg2, ..., an=argn](ActorT *actor) {
// actor->func(a1, a2, ..., an)
// }
//
// But there are some difficulties with elimitation on unnecessary copies.
// We want to use move constructor when it is possible
//
// We may pass
// Tmp. Temporary / rvalue reference
// Var. Variable / reference
// CnstRef. const reference
//
//
// Function may expect
// Val. Value
// CnstRef. const reference
// Ref. rvalue reverence / reference
//
// TODO:
// Immediate call / Delayed call
// Tmp->Val move / move->move
// Tmp->CnstRef + / move->+
// Tmp->Ref + / move->+
// Var->Val copy / copy->move
// Var->CnstRef + / copy->
// Var->Ref + / copy->+ // khm. It will complile, but won't work
//
// So I will use common idiom: forward references
// If delay is needed, just std::forward data to temporary storage, and std::move them when call is executed.
//
//
// create_immediate_closure(&ActorT::func, arg1, arg2, ..., argn).run(actor)
namespace td {
template <class ActorT, class FunctionT, class... ArgsT>
class DelayedClosure;
template <class ActorT, class FunctionT, class... ArgsT>
class ImmediateClosure {
public:
using Delayed = DelayedClosure<ActorT, FunctionT, ArgsT...>;
friend Delayed;
using ActorType = ActorT;
// no &&. just save references as references.
explicit ImmediateClosure(FunctionT func, ArgsT... args) : args(func, std::forward<ArgsT>(args)...) {
}
private:
std::tuple<FunctionT, ArgsT...> args;
public:
auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) {
return mem_call_tuple(actor, std::move(args));
}
};
template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT>
ImmediateClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...> create_immediate_closure(
ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) {
return ImmediateClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...>(func,
std::forward<SrcArgsT>(args)...);
}
template <class ActorT, class FunctionT, class... ArgsT>
class DelayedClosure {
public:
using ActorType = ActorT;
explicit DelayedClosure(ImmediateClosure<ActorT, FunctionT, ArgsT...> &&other) : args(std::move(other.args)) {
}
explicit DelayedClosure(FunctionT func, ArgsT... args) : args(func, std::forward<ArgsT>(args)...) {
}
template <class F>
void for_each(const F &f) {
tuple_for_each(args, f);
}
private:
std::tuple<FunctionT, typename std::decay<ArgsT>::type...> args;
public:
auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) {
return mem_call_tuple(actor, std::move(args));
}
};
template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT>
auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) {
return DelayedClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...>(func,
std::forward<SrcArgsT>(args)...);
}
} // namespace td

86
td/tdutils/td/utils/CombinedLog.h Normal file
View File

@@ -0,0 +1,86 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/algorithm.h"
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/Slice.h"
namespace td {
class CombinedLog final : public LogInterface {
public:
void set_first(LogInterface *first) {
first_ = first;
}
void set_second(LogInterface *second) {
second_ = second;
}
void set_first_verbosity_level(int new_verbosity_level) {
first_verbosity_level_ = new_verbosity_level;
}
void set_second_verbosity_level(int new_verbosity_level) {
second_verbosity_level_ = new_verbosity_level;
}
const LogInterface *get_first() const {
return first_;
}
const LogInterface *get_second() const {
return second_;
}
int get_first_verbosity_level() const {
return first_verbosity_level_;
}
int get_second_verbosity_level() const {
return second_verbosity_level_;
}
private:
LogInterface *first_ = nullptr;
int first_verbosity_level_ = VERBOSITY_NAME(FATAL);
LogInterface *second_ = nullptr;
int second_verbosity_level_ = VERBOSITY_NAME(FATAL);
void do_append(int log_level, CSlice slice) final {
if (first_ && log_level <= first_verbosity_level_) {
first_->do_append(log_level, slice);
}
if (second_ && log_level <= second_verbosity_level_) {
second_->do_append(log_level, slice);
}
}
void after_rotation() final {
if (first_) {
first_->after_rotation();
}
if (second_) {
second_->after_rotation();
}
}
vector<string> get_file_paths() final {
vector<string> result;
if (first_) {
::td::append(result, first_->get_file_paths());
}
if (second_) {
::td::append(result, second_->get_file_paths());
}
return result;
}
};
} // namespace td

321
td/tdutils/td/utils/ConcurrentHashTable.h Normal file
View File

@@ -0,0 +1,321 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/HazardPointers.h"
#include "td/utils/logging.h"
#include "td/utils/port/thread_local.h"
#include <atomic>
#include <condition_variable>
#include <mutex>
namespace td {
// AtomicHashArray<KeyT, ValueT>
// Building block for other concurrent hash maps
//
// Support one operation:
// template <class F>
// bool with_value(KeyT key, bool should_create, F &&func);
//
// Finds slot for key, and call func(value)
// Creates slot if should_create is true.
// Returns true if func was called.
//
// Concurrent calls with the same key may result in concurrent calls to func(value)
// It is responsibility of the caller to handle such races.
//
// Key should already be random
// It is responsibility of the caller to provide unique random key.
// One may use injective hash function, or handle collisions in some other way.
template <class KeyT, class ValueT>
class AtomicHashArray {
public:
explicit AtomicHashArray(size_t n) : nodes_(n) {
}
struct Node {
std::atomic<KeyT> key{KeyT{}};
ValueT value{};
};
size_t size() const {
return nodes_.size();
}
Node &node_at(size_t i) {
return nodes_[i];
}
static KeyT empty_key() {
return KeyT{};
}
template <class F>
bool with_value(KeyT key, bool should_create, F &&f) {
DCHECK(key != empty_key());
auto pos = static_cast<size_t>(key) % nodes_.size();
auto n = td::min(td::max(static_cast<size_t>(300), nodes_.size() / 16 + 2), nodes_.size());
for (size_t i = 0; i < n; i++) {
pos++;
if (pos >= nodes_.size()) {
pos = 0;
}
auto &node = nodes_[pos];
while (true) {
auto node_key = node.key.load(std::memory_order_acquire);
if (node_key == empty_key()) {
if (!should_create) {
return false;
}
KeyT expected_key = empty_key();
if (node.key.compare_exchange_strong(expected_key, key, std::memory_order_relaxed,
std::memory_order_relaxed)) {
f(node.value);
return true;
}
} else if (node_key == key) {
f(node.value);
return true;
} else {
break;
}
}
}
return false;
}
private:
std::vector<Node> nodes_;
};
// Simple concurrent hash map with multiple limitations
template <class KeyT, class ValueT>
class ConcurrentHashMap {
using HashMap = AtomicHashArray<KeyT, std::atomic<ValueT>>;
static HazardPointers<HashMap> hp_;
public:
explicit ConcurrentHashMap(size_t n = 32) {
n = 1;
hash_map_.store(make_unique<HashMap>(n).release());
}
ConcurrentHashMap(const ConcurrentHashMap &) = delete;
ConcurrentHashMap &operator=(const ConcurrentHashMap &) = delete;
ConcurrentHashMap(ConcurrentHashMap &&) = delete;
ConcurrentHashMap &operator=(ConcurrentHashMap &&) = delete;
~ConcurrentHashMap() {
unique_ptr<HashMap>(hash_map_.load()).reset();
}
static std::string get_name() {
return "ConcurrentHashMap";
}
static KeyT empty_key() {
return KeyT{};
}
static ValueT empty_value() {
return ValueT{};
}
static ValueT migrate_value() {
return (ValueT)(1); // c-style conversion because reinterpret_cast<int>(1) is CE in MSVC
}
ValueT insert(KeyT key, ValueT value) {
CHECK(key != empty_key());
CHECK(value != migrate_value());
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
while (true) {
auto hash_map = holder.protect(hash_map_);
if (!hash_map) {
do_migrate(nullptr);
continue;
}
bool ok = false;
ValueT inserted_value;
hash_map->with_value(key, true, [&](auto &node_value) {
ValueT expected_value = this->empty_value();
if (node_value.compare_exchange_strong(expected_value, value, std::memory_order_release,
std::memory_order_acquire)) {
ok = true;
inserted_value = value;
} else {
if (expected_value == this->migrate_value()) {
ok = false;
} else {
ok = true;
inserted_value = expected_value;
}
}
});
if (ok) {
return inserted_value;
}
do_migrate(hash_map);
}
}
ValueT find(KeyT key, ValueT value) {
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
while (true) {
auto hash_map = holder.protect(hash_map_);
if (!hash_map) {
do_migrate(nullptr);
continue;
}
bool has_value = hash_map->with_value(
key, false, [&](auto &node_value) { value = node_value.load(std::memory_order_acquire); });
if (!has_value || value != migrate_value()) {
return value;
}
do_migrate(hash_map);
}
}
template <class F>
void for_each(F &&f) {
auto hash_map = hash_map_.load();
CHECK(hash_map);
auto size = hash_map->size();
for (size_t i = 0; i < size; i++) {
auto &node = hash_map->node_at(i);
auto key = node.key.load(std::memory_order_relaxed);
auto value = node.value.load(std::memory_order_relaxed);
if (key != empty_key()) {
CHECK(value != migrate_value());
if (value != empty_value()) {
f(key, value);
}
}
}
}
private:
// use no padding intentionally
std::atomic<HashMap *> hash_map_{nullptr};
std::mutex migrate_mutex_;
std::condition_variable migrate_cv_;
int migrate_cnt_{0};
int migrate_generation_{0};
HashMap *migrate_from_hash_map_{nullptr};
HashMap *migrate_to_hash_map_{nullptr};
struct Task {
size_t begin;
size_t end;
bool empty() const {
return begin >= end;
}
size_t size() const {
if (empty()) {
return 0;
}
return end - begin;
}
};
struct TaskCreator {
size_t chunk_size;
size_t size;
std::atomic<size_t> pos{0};
Task create() {
auto i = pos++;
auto begin = i * chunk_size;
auto end = begin + chunk_size;
if (end > size) {
end = size;
}
return {begin, end};
}
};
TaskCreator task_creator;
void do_migrate(HashMap *ptr) {
//LOG(ERROR) << "In do_migrate: " << ptr;
std::unique_lock<std::mutex> lock(migrate_mutex_);
if (hash_map_.load() != ptr) {
return;
}
init_migrate();
CHECK(!ptr || migrate_from_hash_map_ == ptr);
migrate_cnt_++;
auto migrate_generation = migrate_generation_;
lock.unlock();
run_migrate();
lock.lock();
migrate_cnt_--;
if (migrate_cnt_ == 0) {
finish_migrate();
}
migrate_cv_.wait(lock, [&] { return migrate_generation_ != migrate_generation; });
}
void finish_migrate() {
//LOG(ERROR) << "In finish_migrate";
hash_map_.store(migrate_to_hash_map_);
hp_.retire(get_thread_id(), migrate_from_hash_map_);
migrate_from_hash_map_ = nullptr;
migrate_to_hash_map_ = nullptr;
migrate_generation_++;
migrate_cv_.notify_all();
}
void init_migrate() {
if (migrate_from_hash_map_ != nullptr) {
return;
}
//LOG(ERROR) << "In init_migrate";
CHECK(migrate_cnt_ == 0);
migrate_generation_++;
migrate_from_hash_map_ = hash_map_.exchange(nullptr);
auto new_size = migrate_from_hash_map_->size() * 2;
migrate_to_hash_map_ = make_unique<HashMap>(new_size).release();
task_creator.chunk_size = 100;
task_creator.size = migrate_from_hash_map_->size();
task_creator.pos = 0;
}
void run_migrate() {
//LOG(ERROR) << "In run_migrate";
size_t cnt = 0;
while (true) {
auto task = task_creator.create();
cnt += task.size();
if (task.empty()) {
break;
}
run_task(task);
}
//LOG(ERROR) << "In run_migrate " << cnt;
}
void run_task(Task task) {
for (auto i = task.begin; i < task.end; i++) {
auto &node = migrate_from_hash_map_->node_at(i);
auto old_value = node.value.exchange(migrate_value(), std::memory_order_acq_rel);
if (old_value == 0) {
continue;
}
auto node_key = node.key.load(std::memory_order_relaxed);
auto ok = migrate_to_hash_map_->with_value(
node_key, true, [&](auto &node_value) { node_value.store(old_value, std::memory_order_relaxed); });
LOG_CHECK(ok) << "Migration overflow";
}
}
};
template <class KeyT, class ValueT>
HazardPointers<typename ConcurrentHashMap<KeyT, ValueT>::HashMap> ConcurrentHashMap<KeyT, ValueT>::hp_(64);
} // namespace td

168
td/tdutils/td/utils/Container.h Normal file
View File

@@ -0,0 +1,168 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <limits>
namespace td {
// 1. Allocates all objects in vector. (but vector never shrinks)
// 2. Id is safe way to reach this object.
// 3. All ids are unique.
// 4. All ids are non-zero.
template <class DataT>
class Container {
public:
using Id = uint64;
DataT *get(Id id) {
int32 slot_id = decode_id(id);
if (slot_id == -1) {
return nullptr;
}
return &slots_[slot_id].data;
}
const DataT *get(Id id) const {
int32 slot_id = decode_id(id);
if (slot_id == -1) {
return nullptr;
}
return &slots_[slot_id].data;
}
void erase(Id id) {
int32 slot_id = decode_id(id);
if (slot_id == -1) {
return;
}
release(slot_id);
}
DataT extract(Id id) {
int32 slot_id = decode_id(id);
CHECK(slot_id != -1);
auto res = std::move(slots_[slot_id].data);
release(slot_id);
return res;
}
Id create(DataT &&data = DataT(), uint8 type = 0) {
int32 id = store(std::move(data), type);
return encode_id(id);
}
Id reset_id(Id id) {
int32 slot_id = decode_id(id);
CHECK(slot_id != -1);
inc_generation(slot_id);
return encode_id(slot_id);
}
static uint8 type_from_id(Id id) {
return static_cast<uint8>(id);
}
vector<Id> ids() const {
vector<bool> is_bad(slots_.size(), false);
for (auto id : empty_slots_) {
is_bad[id] = true;
}
vector<Id> res;
for (size_t i = 0, n = slots_.size(); i < n; i++) {
if (!is_bad[i]) {
res.push_back(encode_id(static_cast<int32>(i)));
}
}
return res;
}
template <class F>
void for_each(const F &f) {
auto ids = this->ids();
for (auto id : ids) {
f(id, *get(id));
}
}
template <class F>
void for_each(const F &f) const {
auto ids = this->ids();
for (auto id : ids) {
f(id, *get(id));
}
}
size_t size() const {
CHECK(empty_slots_.size() <= slots_.size());
return slots_.size() - empty_slots_.size();
}
bool empty() const {
return size() == 0;
}
void clear() {
*this = Container<DataT>();
}
private:
static constexpr uint32 GENERATION_STEP = 1 << 8;
static constexpr uint32 TYPE_MASK = (1 << 8) - 1;
struct Slot {
uint32 generation;
DataT data;
};
vector<Slot> slots_;
vector<int32> empty_slots_;
Id encode_id(int32 id) const {
return (static_cast<uint64>(id) << 32) | slots_[id].generation;
}
int32 decode_id(Id id) const {
auto slot_id = static_cast<int32>(id >> 32);
auto generation = static_cast<uint32>(id);
if (slot_id < 0 || slot_id >= static_cast<int32>(slots_.size())) {
return -1;
}
if (generation != slots_[slot_id].generation) {
return -1;
}
return slot_id;
}
int32 store(DataT &&data, uint8 type) {
int32 pos;
if (!empty_slots_.empty()) {
pos = empty_slots_.back();
empty_slots_.pop_back();
slots_[pos].data = std::move(data);
slots_[pos].generation ^= (slots_[pos].generation & TYPE_MASK) ^ type;
} else {
CHECK(slots_.size() <= static_cast<size_t>(std::numeric_limits<int32>::max()));
pos = static_cast<int32>(slots_.size());
slots_.push_back(Slot{GENERATION_STEP + type, std::move(data)});
}
return pos;
}
void release(int32 id) {
inc_generation(id);
slots_[id].data = DataT();
if (slots_[id].generation & ~TYPE_MASK) { // generation overflow. Can't use this identifier anymore
empty_slots_.push_back(id);
}
}
void inc_generation(int32 id) {
slots_[id].generation += GENERATION_STEP;
}
};
} // namespace td

44
td/tdutils/td/utils/Context.h Normal file
View File

@@ -0,0 +1,44 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/port/thread_local.h"
namespace td {
template <class Impl>
class Context {
public:
static Impl *get() {
return context_;
}
class Guard {
public:
explicit Guard(Impl *new_context) {
old_context_ = context_;
context_ = new_context;
}
~Guard() {
context_ = old_context_;
}
Guard(const Guard &) = delete;
Guard &operator=(const Guard &) = delete;
Guard(Guard &&) = delete;
Guard &operator=(Guard &&) = delete;
private:
Impl *old_context_;
};
private:
static TD_THREAD_LOCAL Impl *context_;
};
template <class Impl>
TD_THREAD_LOCAL Impl *Context<Impl>::context_;
} // namespace td

216
td/tdutils/td/utils/DecTree.h Normal file
View File

@@ -0,0 +1,216 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Random.h"
#include <functional>
#include <utility>
namespace td {
template <typename KeyType, typename ValueType, typename Compare = std::less<KeyType>>
class DecTree {
struct Node {
unique_ptr<Node> left_;
unique_ptr<Node> right_;
size_t size_;
KeyType key_;
ValueType value_;
uint32 y_;
void relax() {
size_ = 1;
if (left_ != nullptr) {
size_ += left_->size_;
}
if (right_ != nullptr) {
size_ += right_->size_;
}
}
Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) {
}
};
unique_ptr<Node> root_;
static unique_ptr<Node> create_node(KeyType key, ValueType value, uint32 y) {
return make_unique<Node>(std::move(key), std::move(value), y);
}
static unique_ptr<Node> insert_node(unique_ptr<Node> Tree, KeyType key, ValueType value, uint32 y) {
if (Tree == nullptr) {
return create_node(std::move(key), std::move(value), y);
}
if (Tree->y_ < y) {
auto P = split_node(std::move(Tree), key);
auto T = create_node(std::move(key), std::move(value), y);
T->left_ = std::move(P.first);
T->right_ = std::move(P.second);
T->relax();
return T;
}
if (Compare()(key, Tree->key_)) {
Tree->left_ = insert_node(std::move(Tree->left_), std::move(key), std::move(value), y);
} else if (Compare()(Tree->key_, key)) {
Tree->right_ = insert_node(std::move(Tree->right_), std::move(key), std::move(value), y);
} else {
// ?? assert
}
Tree->relax();
return Tree;
}
static unique_ptr<Node> remove_node(unique_ptr<Node> Tree, const KeyType &key) {
if (Tree == nullptr) {
// ?? assert
return nullptr;
}
if (Compare()(key, Tree->key_)) {
Tree->left_ = remove_node(std::move(Tree->left_), key);
} else if (Compare()(Tree->key_, key)) {
Tree->right_ = remove_node(std::move(Tree->right_), key);
} else {
Tree = merge_node(std::move(Tree->left_), std::move(Tree->right_));
}
if (Tree != nullptr) {
Tree->relax();
}
return Tree;
}
static ValueType *get_node(unique_ptr<Node> &Tree, const KeyType &key) {
if (Tree == nullptr) {
return nullptr;
}
if (Compare()(key, Tree->key_)) {
return get_node(Tree->left_, key);
} else if (Compare()(Tree->key_, key)) {
return get_node(Tree->right_, key);
} else {
return &Tree->value_;
}
}
static ValueType *get_node_by_idx(unique_ptr<Node> &Tree, size_t idx) {
CHECK(Tree != nullptr);
auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0;
if (idx < s) {
return get_node_by_idx(Tree->left_, idx);
} else if (idx == s) {
return &Tree->value_;
} else {
return get_node_by_idx(Tree->right_, idx - s - 1);
}
}
static const ValueType *get_node(const unique_ptr<Node> &Tree, const KeyType &key) {
if (Tree == nullptr) {
return nullptr;
}
if (Compare()(key, Tree->key_)) {
return get_node(Tree->left_, key);
} else if (Compare()(Tree->key_, key)) {
return get_node(Tree->right_, key);
} else {
return &Tree->value_;
}
}
static const ValueType *get_node_by_idx(const unique_ptr<Node> &Tree, size_t idx) {
CHECK(Tree != nullptr);
auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0;
if (idx < s) {
return get_node_by_idx(Tree->left_, idx);
} else if (idx == s) {
return &Tree->value_;
} else {
return get_node_by_idx(Tree->right_, idx - s - 1);
}
}
static std::pair<unique_ptr<Node>, unique_ptr<Node>> split_node(unique_ptr<Node> Tree, const KeyType &key) {
if (Tree == nullptr) {
return {nullptr, nullptr};
}
if (Compare()(key, Tree->key_)) {
auto P = split_node(std::move(Tree->left_), key);
Tree->left_ = std::move(P.second);
Tree->relax();
P.second = std::move(Tree);
return P;
} else {
auto P = split_node(std::move(Tree->right_), key);
Tree->right_ = std::move(P.first);
Tree->relax();
P.first = std::move(Tree);
return P;
}
}
static unique_ptr<Node> merge_node(unique_ptr<Node> left, unique_ptr<Node> right) {
if (left == nullptr) {
return right;
}
if (right == nullptr) {
return left;
}
if (left->y_ < right->y_) {
right->left_ = merge_node(std::move(left), std::move(right->left_));
right->relax();
return right;
} else {
left->right_ = merge_node(std::move(left->right_), std::move(right));
left->relax();
return left;
}
}
public:
size_t size() const {
if (root_ == nullptr) {
return 0;
} else {
return root_->size_;
}
}
void insert(KeyType key, ValueType value) {
root_ = insert_node(std::move(root_), std::move(key), std::move(value), Random::fast_uint32());
}
void remove(const KeyType &key) {
root_ = remove_node(std::move(root_), key);
}
void reset() {
root_ = nullptr;
}
ValueType *get(const KeyType &key) {
return get_node(root_, key);
}
ValueType *get_random() {
if (size() == 0) {
return nullptr;
} else {
return get_node_by_idx(root_, Random::fast_uint32() % size());
}
}
const ValueType *get(const KeyType &key) const {
return get_node(root_, key);
}
const ValueType *get_random() const {
if (size() == 0) {
return nullptr;
} else {
return get_node_by_idx(root_, Random::fast_uint32() % size());
}
}
bool exists(const KeyType &key) const {
return get_node(root_, key) != nullptr;
}
};
} // namespace td

52
td/tdutils/td/utils/Destructor.h Normal file
View File

@@ -0,0 +1,52 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <memory>
#include <utility>
namespace td {
class Destructor {
public:
Destructor() = default;
Destructor(const Destructor &) = delete;
Destructor &operator=(const Destructor &) = delete;
Destructor(Destructor &&) = default;
Destructor &operator=(Destructor &&) = default;
virtual ~Destructor() = default;
};
template <class F>
class LambdaDestructor final : public Destructor {
public:
explicit LambdaDestructor(F &&f) : f_(std::move(f)) {
}
LambdaDestructor(const LambdaDestructor &) = delete;
LambdaDestructor &operator=(const LambdaDestructor &) = delete;
LambdaDestructor(LambdaDestructor &&) = default;
LambdaDestructor &operator=(LambdaDestructor &&) = default;
~LambdaDestructor() final {
f_();
}
private:
F f_;
};
template <class F>
auto create_destructor(F &&f) {
return make_unique<LambdaDestructor<F>>(std::forward<F>(f));
}
template <class F>
auto create_shared_destructor(F &&f) {
return std::make_shared<LambdaDestructor<F>>(std::forward<F>(f));
}
} // namespace td

55
td/tdutils/td/utils/Enumerator.h Normal file
View File

@@ -0,0 +1,55 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/WaitFreeVector.h"
#include <limits>
#include <map>
#include <tuple>
namespace td {
template <class ValueT>
class Enumerator {
public:
using Key = int32;
Key add(ValueT v) {
CHECK(arr_.size() < static_cast<size_t>(std::numeric_limits<int32>::max() - 1));
auto next_id = static_cast<int32>(arr_.size() + 1);
bool was_inserted;
decltype(map_.begin()) it;
std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id);
if (was_inserted) {
arr_.push_back(&it->first);
}
return it->second;
}
const ValueT &get(Key key) const {
auto pos = static_cast<size_t>(key - 1);
CHECK(pos < arr_.size());
return *arr_[pos];
}
size_t size() const {
CHECK(map_.size() == arr_.size());
return arr_.size();
}
bool empty() const {
return size() == 0;
}
private:
std::map<ValueT, int32> map_;
WaitFreeVector<const ValueT *> arr_;
};
} // namespace td

201
td/tdutils/td/utils/EpochBasedMemoryReclamation.h Normal file
View File

@@ -0,0 +1,201 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/sleep.h"
#include <atomic>
#include <memory>
namespace td {
template <class T>
class EpochBasedMemoryReclamation {
public:
EpochBasedMemoryReclamation(const EpochBasedMemoryReclamation &) = delete;
EpochBasedMemoryReclamation &operator=(const EpochBasedMemoryReclamation &) = delete;
EpochBasedMemoryReclamation(EpochBasedMemoryReclamation &&) = delete;
EpochBasedMemoryReclamation &operator=(EpochBasedMemoryReclamation &&) = delete;
~EpochBasedMemoryReclamation() = default;
class Locker {
public:
Locker(size_t thread_id, EpochBasedMemoryReclamation *ebmr) : thread_id_(thread_id), ebmr_(ebmr) {
}
Locker(const Locker &) = delete;
Locker &operator=(const Locker &) = delete;
Locker(Locker &&) = default;
Locker &operator=(Locker &&) = delete;
~Locker() {
if (ebmr_) {
retire_sync();
unlock();
(void)ebmr_.release();
}
}
void lock() {
DCHECK(ebmr_);
ebmr_->lock(thread_id_);
}
void unlock() {
DCHECK(ebmr_);
ebmr_->unlock(thread_id_);
}
void retire_sync() {
ebmr_->retire_sync(thread_id_);
}
void retire() {
ebmr_->retire(thread_id_);
}
void retire(T *ptr) {
ebmr_->retire(thread_id_, ptr);
}
private:
size_t thread_id_;
struct Never {
template <class S>
void operator()(S *) const {
UNREACHABLE();
}
};
std::unique_ptr<EpochBasedMemoryReclamation, Never> ebmr_;
};
explicit EpochBasedMemoryReclamation(size_t threads_n) : threads_(threads_n) {
}
Locker get_locker(size_t thread_id) {
return Locker{thread_id, this};
}
size_t to_delete_size_unsafe() const {
size_t res = 0;
for (auto &thread_data : threads_) {
// LOG(ERROR) << "---" << thread_data.epoch.load() / 2;
for (size_t i = 0; i < MAX_BAGS; i++) {
res += thread_data.to_delete[i].size();
// LOG(ERROR) << thread_data.to_delete[i].size();
}
}
return res;
}
private:
static constexpr size_t MAX_BAGS = 3;
struct ThreadData {
std::atomic<int64> epoch{1};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<int64>)];
size_t to_skip{0};
size_t checked_thread_i{0};
size_t bag_i{0};
std::vector<unique_ptr<T>> to_delete[MAX_BAGS];
char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<unique_ptr<T>>) * MAX_BAGS];
void rotate_bags() {
bag_i = (bag_i + 1) % MAX_BAGS;
to_delete[bag_i].clear();
}
void set_epoch(int64 new_epoch) {
//LOG(ERROR) << new_epoch;
if (epoch.load(std::memory_order_relaxed) / 2 != new_epoch) {
checked_thread_i = 0;
to_skip = 0;
rotate_bags();
}
epoch = new_epoch * 2;
}
void idle() {
epoch.store(epoch.load(std::memory_order_relaxed) | 1);
}
size_t undeleted() const {
size_t res = 0;
for (size_t i = 0; i < MAX_BAGS; i++) {
res += to_delete[i].size();
}
return res;
}
};
std::vector<ThreadData> threads_;
char pad[TD_CONCURRENCY_PAD - sizeof(std::vector<ThreadData>)];
std::atomic<int64> epoch_{1};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<int64>)];
void lock(size_t thread_id) {
auto &data = threads_[thread_id];
auto epoch = epoch_.load();
data.set_epoch(epoch);
if (data.to_skip == 0) {
data.to_skip = 30;
step_check(data);
} else {
data.to_skip--;
}
}
void unlock(size_t thread_id) {
//LOG(ERROR) << "UNLOCK";
auto &data = threads_[thread_id];
data.idle();
}
bool step_check(ThreadData &data) {
auto epoch = data.epoch.load(std::memory_order_relaxed) / 2;
auto checked_thread_epoch = threads_[data.checked_thread_i].epoch.load();
if (checked_thread_epoch % 2 == 1 || checked_thread_epoch / 2 == epoch) {
data.checked_thread_i++;
if (data.checked_thread_i == threads_.size()) {
if (epoch_.compare_exchange_strong(epoch, epoch + 1)) {
data.set_epoch(epoch + 1);
} else {
data.set_epoch(epoch);
}
}
return true;
}
return false;
}
void retire_sync(size_t thread_id) {
auto &data = threads_[thread_id];
while (true) {
retire(thread_id);
data.idle();
if (data.undeleted() == 0) {
break;
}
usleep_for(1000);
}
}
void retire(size_t thread_id) {
auto &data = threads_[thread_id];
data.set_epoch(epoch_.load());
while (step_check(data) && data.undeleted() != 0) {
}
}
void retire(size_t thread_id, T *ptr) {
auto &data = threads_[thread_id];
data.to_delete[data.bag_i].push_back(unique_ptr<T>{ptr});
}
};
} // namespace td

20
td/tdutils/td/utils/ExitGuard.cpp Normal file
View File

@@ -0,0 +1,20 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/ExitGuard.h"
#include "td/utils/logging.h"
namespace td {
std::atomic<bool> ExitGuard::is_exited_{false};
ExitGuard::~ExitGuard() {
is_exited_.store(true, std::memory_order_relaxed);
set_verbosity_level(VERBOSITY_NAME(FATAL));
}
} // namespace td

30
td/tdutils/td/utils/ExitGuard.h Normal file
View File

@@ -0,0 +1,30 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include <atomic>
namespace td {
class ExitGuard {
public:
ExitGuard() = default;
ExitGuard(ExitGuard &&) = delete;
ExitGuard &operator=(ExitGuard &&) = delete;
ExitGuard(const ExitGuard &) = delete;
ExitGuard &operator=(const ExitGuard &) = delete;
~ExitGuard();
static bool is_exited() {
return is_exited_.load(std::memory_order_relaxed);
}
private:
static std::atomic<bool> is_exited_;
};
} // namespace td

145
td/tdutils/td/utils/FileLog.cpp Normal file
View File

@@ -0,0 +1,145 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/FileLog.h"
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/port/path.h"
#include "td/utils/port/StdStreams.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/Slice.h"
#include "td/utils/SliceBuilder.h"
#include "td/utils/Time.h"
namespace td {
Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) {
if (path.empty()) {
return Status::Error("Log file path must be non-empty");
}
if (path == path_) {
set_rotate_threshold(rotate_threshold);
return Status::OK();
}
TRY_RESULT(fd, FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append));
fd_.close();
fd_ = std::move(fd);
if (!Stderr().empty() && redirect_stderr) {
fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
auto r_path = realpath(path, true);
if (r_path.is_error()) {
path_ = std::move(path);
} else {
path_ = r_path.move_as_ok();
}
TRY_RESULT_ASSIGN(size_, fd_.get_size());
rotate_threshold_ = rotate_threshold;
redirect_stderr_ = redirect_stderr;
return Status::OK();
}
Slice FileLog::get_path() const {
return path_;
}
vector<string> FileLog::get_file_paths() {
vector<string> result;
if (!path_.empty()) {
result.push_back(path_);
result.push_back(PSTRING() << path_ << ".old");
}
return result;
}
void FileLog::set_rotate_threshold(int64 rotate_threshold) {
rotate_threshold_ = rotate_threshold;
}
int64 FileLog::get_rotate_threshold() const {
return rotate_threshold_;
}
bool FileLog::get_redirect_stderr() const {
return redirect_stderr_;
}
void FileLog::do_append(int log_level, CSlice slice) {
auto start_time = Time::now();
if (size_ > rotate_threshold_ || want_rotate_.load(std::memory_order_relaxed)) {
auto status = rename(path_, PSLICE() << path_ << ".old");
if (status.is_error()) {
process_fatal_error(PSLICE() << status << " in " << __FILE__ << " at " << __LINE__ << '\n');
}
do_after_rotation();
}
while (!slice.empty()) {
if (redirect_stderr_) {
while (has_log_guard()) {
// spin
}
}
auto r_size = fd_.write(slice);
if (r_size.is_error()) {
process_fatal_error(PSLICE() << r_size.error() << " in " << __FILE__ << " at " << __LINE__ << '\n');
}
auto written = r_size.ok();
size_ += static_cast<int64>(written);
slice.remove_prefix(written);
}
auto total_time = Time::now() - start_time;
if (total_time >= 0.1 && log_level >= 1) {
auto thread_id = get_thread_id();
auto r_size = fd_.write(PSLICE() << "[ 2][t" << (0 <= thread_id && thread_id < 10 ? " " : "") << thread_id
<< "] !!! Previous logging took " << total_time << " seconds !!!\n");
r_size.ignore();
}
}
void FileLog::after_rotation() {
if (path_.empty()) {
return;
}
do_after_rotation();
}
void FileLog::lazy_rotate() {
want_rotate_ = true;
}
void FileLog::do_after_rotation() {
want_rotate_ = false;
ScopedDisableLog disable_log; // to ensure that nothing will be printed to the closed log
CHECK(!path_.empty());
fd_.close();
auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Write | FileFd::Append);
if (r_fd.is_error()) {
process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__ << '\n');
}
fd_ = r_fd.move_as_ok();
if (!Stderr().empty() && redirect_stderr_) {
fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
auto r_size = fd_.get_size();
if (r_fd.is_error()) {
process_fatal_error(PSLICE() << "Failed to get log size: " << r_fd.error() << " in " << __FILE__ << " at "
<< __LINE__ << '\n');
}
size_ = r_size.move_as_ok();
}
Result<unique_ptr<LogInterface>> FileLog::create(string path, int64 rotate_threshold, bool redirect_stderr) {
auto l = make_unique<FileLog>();
TRY_STATUS(l->init(std::move(path), rotate_threshold, redirect_stderr));
return std::move(l);
}
} // namespace td

54
td/tdutils/td/utils/FileLog.h Normal file
View File

@@ -0,0 +1,54 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include <atomic>
namespace td {
class FileLog final : public LogInterface {
static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20);
public:
static Result<unique_ptr<LogInterface>> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD,
bool redirect_stderr = true);
Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true);
Slice get_path() const;
vector<string> get_file_paths() final;
void set_rotate_threshold(int64 rotate_threshold);
int64 get_rotate_threshold() const;
bool get_redirect_stderr() const;
void after_rotation() final;
void lazy_rotate();
private:
FileFd fd_;
string path_;
int64 size_ = 0;
int64 rotate_threshold_ = 0;
bool redirect_stderr_ = false;
std::atomic<bool> want_rotate_{false};
void do_append(int log_level, CSlice slice) final;
void do_after_rotation();
};
} // namespace td

24
td/tdutils/td/utils/FlatHashMap.h Normal file
View File

@@ -0,0 +1,24 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
//#include "td/utils/FlatHashMapChunks.h"
#include "td/utils/FlatHashTable.h"
#include "td/utils/HashTableUtils.h"
#include "td/utils/MapNode.h"
#include <functional>
//#include <unordered_map>
namespace td {
template <class KeyT, class ValueT, class HashT = Hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashMap = FlatHashTable<MapNode<KeyT, ValueT, EqT>, HashT, EqT>;
//using FlatHashMap = FlatHashMapChunks<KeyT, ValueT, HashT, EqT>;
//using FlatHashMap = std::unordered_map<KeyT, ValueT, HashT, EqT>;
} // namespace td

578
td/tdutils/td/utils/FlatHashMapChunks.h Normal file
View File

@@ -0,0 +1,578 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/bits.h"
#include "td/utils/common.h"
#include "td/utils/fixed_vector.h"
#include "td/utils/HashTableUtils.h"
#include "td/utils/MapNode.h"
#include "td/utils/SetNode.h"
#include <cstddef>
#include <functional>
#include <initializer_list>
#include <iterator>
#include <limits>
#include <utility>
#if defined(__SSE2__) || (TD_MSVC && (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)))
#define TD_SSE2 1
#endif
#ifdef __aarch64__
#include <arm_neon.h>
#endif
#if TD_SSE2
#include <emmintrin.h>
#endif
namespace td {
template <int shift>
struct MaskIterator {
uint64 mask;
explicit operator bool() const noexcept {
return mask != 0;
}
int pos() const {
return count_trailing_zeroes64(mask) / shift;
}
void next() {
mask &= mask - 1;
}
// For foreach
bool operator!=(MaskIterator &other) const {
return mask != other.mask;
}
auto operator*() const {
return pos();
}
void operator++() {
next();
}
auto begin() {
return *this;
}
auto end() {
return MaskIterator{0u};
}
};
struct MaskPortable {
static MaskIterator<1> equal_mask(uint8 *bytes, uint8 needle) {
uint64 res = 0;
for (int i = 0; i < 16; i++) {
res |= (bytes[i] == needle) << i;
}
return {res & ((1u << 14) - 1)};
}
};
#ifdef __aarch64__
struct MaskNeonFolly {
static MaskIterator<4> equal_mask(uint8 *bytes, uint8 needle) {
uint8x16_t input_mask = vld1q_u8(bytes);
auto needle_mask = vdupq_n_u8(needle);
auto eq_mask = vceqq_u8(input_mask, needle_mask);
// get info from every byte into the bottom half of every uint16
// by shifting right 4, then round to get it into a 64-bit vector
uint8x8_t shifted_eq_mask = vshrn_n_u16(vreinterpretq_u16_u8(eq_mask), 4);
uint64 mask = vget_lane_u64(vreinterpret_u64_u8(shifted_eq_mask), 0);
return {mask & 0x11111111111111};
}
};
struct MaskNeon {
static MaskIterator<1> equal_mask(uint8 *bytes, uint8 needle) {
uint8x16_t input_mask = vld1q_u8(bytes);
auto needle_mask = vdupq_n_u8(needle);
auto eq_mask = vceqq_u8(input_mask, needle_mask);
uint16x8_t MASK = vdupq_n_u16(0x180);
uint16x8_t a_masked = vandq_u16(vreinterpretq_u16_u8(eq_mask), MASK);
const int16 __attribute__((aligned(16))) SHIFT_ARR[8] = {-7, -5, -3, -1, 1, 3, 5, 7};
int16x8_t SHIFT = vld1q_s16(SHIFT_ARR);
uint16x8_t a_shifted = vshlq_u16(a_masked, SHIFT);
return {vaddvq_u16(a_shifted) & ((1u << 14) - 1)};
}
};
#elif TD_SSE2
struct MaskSse2 {
static MaskIterator<1> equal_mask(uint8 *bytes, uint8 needle) {
auto input_mask = _mm_loadu_si128(reinterpret_cast<const __m128i *>(bytes));
auto needle_mask = _mm_set1_epi8(needle);
auto match_mask = _mm_cmpeq_epi8(needle_mask, input_mask);
return {static_cast<uint32>(_mm_movemask_epi8(match_mask)) & ((1u << 14) - 1)};
}
};
#endif
#ifdef __aarch64__
using MaskHelper = MaskNeonFolly;
#elif TD_SSE2
using MaskHelper = MaskSse2;
#else
using MaskHelper = MaskPortable;
#endif
template <class NodeT, class HashT, class EqT>
class FlatHashTableChunks {
public:
using Self = FlatHashTableChunks<NodeT, HashT, EqT>;
using Node = NodeT;
using NodeIterator = typename fixed_vector<Node>::iterator;
using ConstNodeIterator = typename fixed_vector<Node>::const_iterator;
using KeyT = typename Node::public_key_type;
using key_type = typename Node::public_key_type;
using value_type = typename Node::public_type;
struct Iterator {
using iterator_category = std::bidirectional_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = FlatHashTableChunks::value_type;
using pointer = value_type *;
using reference = value_type &;
friend class FlatHashTableChunks;
Iterator &operator++() {
do {
++it_;
} while (it_ != map_->nodes_.end() && it_->empty());
return *this;
}
Iterator &operator--() {
do {
--it_;
} while (it_->empty());
return *this;
}
reference operator*() {
return it_->get_public();
}
pointer operator->() {
return &it_->get_public();
}
bool operator==(const Iterator &other) const {
DCHECK(map_ == other.map_);
return it_ == other.it_;
}
bool operator!=(const Iterator &other) const {
DCHECK(map_ == other.map_);
return it_ != other.it_;
}
Iterator() = default;
Iterator(NodeIterator it, Self *map) : it_(std::move(it)), map_(map) {
}
private:
NodeIterator it_;
Self *map_;
};
struct ConstIterator {
using iterator_category = std::bidirectional_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = FlatHashTableChunks::value_type;
using pointer = const value_type *;
using reference = const value_type &;
friend class FlatHashTableChunks;
ConstIterator &operator++() {
++it_;
return *this;
}
ConstIterator &operator--() {
--it_;
return *this;
}
reference operator*() {
return *it_;
}
pointer operator->() {
return &*it_;
}
bool operator==(const ConstIterator &other) const {
return it_ == other.it_;
}
bool operator!=(const ConstIterator &other) const {
return it_ != other.it_;
}
ConstIterator() = default;
ConstIterator(Iterator it) : it_(std::move(it)) {
}
private:
Iterator it_;
};
using iterator = Iterator;
using const_iterator = ConstIterator;
FlatHashTableChunks() = default;
FlatHashTableChunks(const FlatHashTableChunks &other) {
assign(other);
}
FlatHashTableChunks &operator=(const FlatHashTableChunks &other) {
clear();
assign(other);
return *this;
}
FlatHashTableChunks(std::initializer_list<Node> nodes) {
reserve(nodes.size());
for (auto &new_node : nodes) {
CHECK(!new_node.empty());
if (count(new_node.key()) > 0) {
continue;
}
Node node;
node.copy_from(new_node);
emplace_node(std::move(node));
}
}
FlatHashTableChunks(FlatHashTableChunks &&other) noexcept {
swap(other);
}
FlatHashTableChunks &operator=(FlatHashTableChunks &&other) noexcept {
swap(other);
return *this;
}
void swap(FlatHashTableChunks &other) noexcept {
nodes_.swap(other.nodes_);
chunks_.swap(other.chunks_);
std::swap(used_nodes_, other.used_nodes_);
}
~FlatHashTableChunks() = default;
size_t bucket_count() const {
return nodes_.size();
}
Iterator find(const KeyT &key) {
if (empty() || is_hash_table_key_empty<EqT>(key)) {
return end();
}
const auto hash = calc_hash(key);
auto chunk_it = get_chunk_it(hash.chunk_i);
while (true) {
auto chunk_i = chunk_it.pos();
auto chunk_begin = nodes_.begin() + chunk_i * Chunk::CHUNK_SIZE;
//__builtin_prefetch(chunk_begin);
auto &chunk = chunks_[chunk_i];
auto mask_it = MaskHelper::equal_mask(chunk.ctrl, hash.small_hash);
for (auto pos : mask_it) {
auto it = chunk_begin + pos;
if (likely(EqT()(it->key(), key))) {
return Iterator{it, this};
}
}
if (chunk.skipped_cnt == 0) {
break;
}
chunk_it.next();
}
return end();
}
ConstIterator find(const KeyT &key) const {
return ConstIterator(const_cast<Self *>(this)->find(key));
}
size_t size() const {
return used_nodes_;
}
bool empty() const {
return size() == 0;
}
Iterator begin() {
if (empty()) {
return end();
}
auto it = nodes_.begin();
while (it->empty()) {
++it;
}
return Iterator(it, this);
}
Iterator end() {
return Iterator(nodes_.end(), this);
}
ConstIterator begin() const {
return ConstIterator(const_cast<Self *>(this)->begin());
}
ConstIterator end() const {
return ConstIterator(const_cast<Self *>(this)->end());
}
void reserve(size_t size) {
//size_t want_size = normalize(size * 5 / 3 + 1);
size_t want_size = normalize(size * 14 / 12 + 1);
// size_t want_size = size * 2;
if (want_size > nodes_.size()) {
resize(want_size);
}
}
template <class... ArgsT>
std::pair<Iterator, bool> emplace(KeyT key, ArgsT &&...args) {
CHECK(!is_hash_table_key_empty<EqT>(key));
auto it = find(key);
if (it != end()) {
return {it, false};
}
try_grow();
auto hash = calc_hash(key);
auto chunk_it = get_chunk_it(hash.chunk_i);
while (true) {
auto chunk_i = chunk_it.pos();
auto &chunk = chunks_[chunk_i];
auto mask_it = MaskHelper::equal_mask(chunk.ctrl, 0);
if (mask_it) {
auto shift = mask_it.pos();
DCHECK(chunk.ctrl[shift] == 0);
auto node_it = nodes_.begin() + shift + chunk_i * Chunk::CHUNK_SIZE;
DCHECK(node_it->empty());
node_it->emplace(std::move(key), std::forward<ArgsT>(args)...);
DCHECK(!node_it->empty());
chunk.ctrl[shift] = hash.small_hash;
used_nodes_++;
return {{node_it, this}, true};
}
CHECK(chunk.skipped_cnt != std::numeric_limits<uint16>::max());
chunk.skipped_cnt++;
chunk_it.next();
}
}
std::pair<Iterator, bool> insert(KeyT key) {
return emplace(std::move(key));
}
template <class ItT>
void insert(ItT begin, ItT end) {
for (; begin != end; ++begin) {
emplace(*begin);
}
}
template <class T = typename Node::second_type>
T &operator[](const KeyT &key) {
return emplace(key).first->second;
}
size_t erase(const KeyT &key) {
auto it = find(key);
if (it == end()) {
return 0;
}
erase(it);
try_shrink();
return 1;
}
size_t count(const KeyT &key) const {
return find(key) != end();
}
void clear() {
used_nodes_ = 0;
nodes_ = {};
chunks_ = {};
}
void erase(Iterator it) {
DCHECK(it != end());
DCHECK(!it.it_->empty());
erase_node(it.it_);
}
template <class F>
bool remove_if(F &&f) {
bool is_removed = false;
for (auto it = nodes_.begin(), end = nodes_.end(); it != end; ++it) {
if (!it->empty() && f(it->get_public())) {
erase_node(it);
is_removed = true;
}
}
try_shrink();
return is_removed;
}
private:
struct Chunk {
static constexpr int CHUNK_SIZE = 14;
static constexpr int MASK = (1 << CHUNK_SIZE) - 1;
// 0x0 - empty
uint8 ctrl[CHUNK_SIZE] = {};
uint16 skipped_cnt{0};
};
fixed_vector<Node> nodes_;
fixed_vector<Chunk> chunks_;
size_t used_nodes_{};
void assign(const FlatHashTableChunks &other) {
reserve(other.size());
for (const auto &new_node : other) {
Node node;
node.copy_from(new_node);
emplace_node(std::move(node));
}
}
void try_grow() {
if (should_grow(used_nodes_ + 1, nodes_.size())) {
grow();
}
}
static bool should_grow(size_t used_count, size_t bucket_count) {
return used_count * 14 > bucket_count * 12;
}
void try_shrink() {
if (should_shrink(used_nodes_, nodes_.size())) {
shrink();
}
}
static bool should_shrink(size_t used_count, size_t bucket_count) {
return used_count * 10 < bucket_count;
}
static size_t normalize(size_t size) {
auto x = (size / Chunk::CHUNK_SIZE) | 1;
auto y = static_cast<size_t>(1) << (64 - count_leading_zeroes64(x));
return y * Chunk::CHUNK_SIZE;
}
void shrink() {
size_t want_size = normalize((used_nodes_ + 1) * 5 / 3 + 1);
resize(want_size);
}
void grow() {
size_t want_size = normalize(2 * nodes_.size() - !nodes_.empty());
resize(want_size);
}
struct HashInfo {
size_t chunk_i;
uint8 small_hash;
};
struct ChunkIt {
size_t chunk_i;
size_t chunk_mask;
size_t shift;
size_t pos() const {
return chunk_i;
}
void next() {
DCHECK((chunk_mask & (chunk_mask + 1)) == 0);
shift++;
chunk_i += shift;
chunk_i &= chunk_mask;
}
};
ChunkIt get_chunk_it(size_t chunk_i) {
return ChunkIt{chunk_i, chunks_.size() - 1, 0};
}
HashInfo calc_hash(const KeyT &key) {
auto h = HashT()(key);
return {(h >> 8) % chunks_.size(), static_cast<uint8>(0x80 | h)};
}
void resize(size_t new_size) {
CHECK(new_size >= Chunk::CHUNK_SIZE);
fixed_vector<Node> old_nodes(new_size);
fixed_vector<Chunk> chunks(new_size / Chunk::CHUNK_SIZE);
old_nodes.swap(nodes_);
chunks_ = std::move(chunks);
used_nodes_ = 0;
for (auto &node : old_nodes) {
if (node.empty()) {
continue;
}
emplace_node(std::move(node));
}
}
void emplace_node(Node &&node) {
DCHECK(!node.empty());
auto hash = calc_hash(node.key());
auto chunk_it = get_chunk_it(hash.chunk_i);
while (true) {
auto chunk_i = chunk_it.pos();
auto &chunk = chunks_[chunk_i];
auto mask_it = MaskHelper::equal_mask(chunk.ctrl, 0);
if (mask_it) {
auto shift = mask_it.pos();
auto node_it = nodes_.begin() + shift + chunk_i * Chunk::CHUNK_SIZE;
DCHECK(node_it->empty());
*node_it = std::move(node);
DCHECK(chunk.ctrl[shift] == 0);
chunk.ctrl[shift] = hash.small_hash;
DCHECK(chunk.ctrl[shift] != 0);
used_nodes_++;
break;
}
CHECK(chunk.skipped_cnt != std::numeric_limits<uint16>::max());
chunk.skipped_cnt++;
chunk_it.next();
}
}
void next_bucket(size_t &bucket) const {
bucket++;
if (unlikely(bucket == nodes_.size())) {
bucket = 0;
}
}
void erase_node(NodeIterator it) {
DCHECK(!it->empty());
size_t empty_i = it - nodes_.begin();
DCHECK(0 <= empty_i && empty_i < nodes_.size());
auto empty_chunk_i = empty_i / Chunk::CHUNK_SIZE;
auto hash = calc_hash(it->key());
auto chunk_it = get_chunk_it(hash.chunk_i);
while (true) {
auto chunk_i = chunk_it.pos();
auto &chunk = chunks_[chunk_i];
if (chunk_i == empty_chunk_i) {
chunk.ctrl[empty_i - empty_chunk_i * Chunk::CHUNK_SIZE] = 0;
break;
}
chunk.skipped_cnt--;
chunk_it.next();
}
it->clear();
used_nodes_--;
}
};
template <class KeyT, class ValueT, class HashT = Hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashMapChunks = FlatHashTableChunks<MapNode<KeyT, ValueT, EqT>, HashT, EqT>;
template <class KeyT, class HashT = Hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashSetChunks = FlatHashTableChunks<SetNode<KeyT, EqT>, HashT, EqT>;
template <class NodeT, class HashT, class EqT, class FuncT>
bool table_remove_if(FlatHashTableChunks<NodeT, HashT, EqT> &table, FuncT &&func) {
return table.remove_if(func);
}
} // namespace td

24
td/tdutils/td/utils/FlatHashSet.h Normal file
View File

@@ -0,0 +1,24 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
//#include "td/utils/FlatHashMapChunks.h"
#include "td/utils/FlatHashTable.h"
#include "td/utils/HashTableUtils.h"
#include "td/utils/SetNode.h"
#include <functional>
//#include <unordered_set>
namespace td {
template <class KeyT, class HashT = Hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashSet = FlatHashTable<SetNode<KeyT, EqT>, HashT, EqT>;
//using FlatHashSet = FlatHashSetChunks<KeyT, HashT, EqT>;
//using FlatHashSet = std::unordered_set<KeyT, HashT, EqT>;
} // namespace td

24
td/tdutils/td/utils/FlatHashTable.cpp Normal file
View File

@@ -0,0 +1,24 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/FlatHashTable.h"
#include "td/utils/bits.h"
#include "td/utils/Random.h"
namespace td {
namespace detail {
uint32 normalize_flat_hash_table_size(uint32 size) {
return td::max(static_cast<uint32>(1) << (32 - count_leading_zeroes32(size)), static_cast<uint32>(8));
}
uint32 get_random_flat_hash_table_bucket(uint32 bucket_count_mask) {
return Random::fast_uint32() & bucket_count_mask;
}
} // namespace detail
} // namespace td

562
td/tdutils/td/utils/FlatHashTable.h Normal file
View File

@@ -0,0 +1,562 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/HashTableUtils.h"
#include <cstddef>
#include <initializer_list>
#include <iterator>
#include <utility>
namespace td {
namespace detail {
uint32 normalize_flat_hash_table_size(uint32 size);
uint32 get_random_flat_hash_table_bucket(uint32 bucket_count_mask);
} // namespace detail
template <class NodeT, class HashT, class EqT>
class FlatHashTable {
static constexpr uint32 INVALID_BUCKET = 0xFFFFFFFF;
void allocate_nodes(uint32 size) {
DCHECK(size >= 8);
DCHECK((size & (size - 1)) == 0);
CHECK(size <= min(static_cast<uint32>(1) << 29, static_cast<uint32>(0x7FFFFFFF / sizeof(NodeT))));
nodes_ = new NodeT[size];
// used_node_count_ = 0;
bucket_count_mask_ = size - 1;
bucket_count_ = size;
begin_bucket_ = INVALID_BUCKET;
}
static void clear_nodes(NodeT *nodes) {
delete[] nodes;
}
public:
using KeyT = typename NodeT::public_key_type;
using key_type = typename NodeT::public_key_type;
using value_type = typename NodeT::public_type;
// TODO use EndSentinel for end() after switching to C++17
// struct EndSentinel {};
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = typename NodeT::public_type;
using pointer = value_type *;
using reference = value_type &;
Iterator &operator++() {
DCHECK(it_ != nullptr);
do {
if (unlikely(++it_ == end_)) {
it_ = begin_;
}
if (unlikely(it_ == start_)) {
it_ = nullptr;
break;
}
} while (it_->empty());
return *this;
}
reference operator*() {
return it_->get_public();
}
const value_type &operator*() const {
return it_->get_public();
}
pointer operator->() {
return &it_->get_public();
}
const value_type *operator->() const {
return &it_->get_public();
}
NodeT *get() {
return it_;
}
bool operator==(const Iterator &other) const {
DCHECK(other.it_ == nullptr);
return it_ == nullptr;
}
bool operator!=(const Iterator &other) const {
DCHECK(other.it_ == nullptr);
return it_ != nullptr;
}
Iterator() = default;
Iterator(NodeT *it, NodeT *begin, NodeT *end) : it_(it), begin_(begin), start_(it), end_(end) {
}
private:
NodeT *it_ = nullptr;
NodeT *begin_ = nullptr;
NodeT *start_ = nullptr;
NodeT *end_ = nullptr;
};
struct ConstIterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = typename NodeT::public_type;
using pointer = const value_type *;
using reference = const value_type &;
ConstIterator &operator++() {
++it_;
return *this;
}
reference operator*() const {
return *it_;
}
pointer operator->() const {
return &*it_;
}
bool operator==(const ConstIterator &other) const {
return it_ == other.it_;
}
bool operator!=(const ConstIterator &other) const {
return it_ != other.it_;
}
ConstIterator() = default;
ConstIterator(Iterator it) : it_(std::move(it)) {
}
private:
Iterator it_;
};
using iterator = Iterator;
using const_iterator = ConstIterator;
struct NodePointer {
value_type &operator*() {
return it_->get_public();
}
const value_type &operator*() const {
return it_->get_public();
}
value_type *operator->() {
return &it_->get_public();
}
const value_type *operator->() const {
return &it_->get_public();
}
NodeT *get() {
return it_;
}
bool operator==(const Iterator &) const {
return it_ == nullptr;
}
bool operator!=(const Iterator &) const {
return it_ != nullptr;
}
explicit NodePointer(NodeT *it) : it_(it) {
}
private:
NodeT *it_ = nullptr;
};
struct ConstNodePointer {
const value_type &operator*() const {
return it_->get_public();
}
const value_type *operator->() const {
return &it_->get_public();
}
bool operator==(const ConstIterator &) const {
return it_ == nullptr;
}
bool operator!=(const ConstIterator &) const {
return it_ != nullptr;
}
const NodeT *get() const {
return it_;
}
explicit ConstNodePointer(const NodeT *it) : it_(it) {
}
private:
const NodeT *it_ = nullptr;
};
FlatHashTable() = default;
FlatHashTable(const FlatHashTable &) = delete;
FlatHashTable &operator=(const FlatHashTable &) = delete;
FlatHashTable(std::initializer_list<NodeT> nodes) {
if (nodes.size() == 0) {
return;
}
reserve(nodes.size());
uint32 used_nodes = 0;
for (auto &new_node : nodes) {
CHECK(!new_node.empty());
auto bucket = calc_bucket(new_node.key());
while (true) {
auto &node = nodes_[bucket];
if (node.empty()) {
node.copy_from(new_node);
used_nodes++;
break;
}
if (EqT()(node.key(), new_node.key())) {
break;
}
next_bucket(bucket);
}
}
used_node_count_ = used_nodes;
}
template <class T>
FlatHashTable(std::initializer_list<T> keys) {
for (auto &key : keys) {
emplace(KeyT(key));
}
}
FlatHashTable(FlatHashTable &&other) noexcept
: nodes_(other.nodes_)
, used_node_count_(other.used_node_count_)
, bucket_count_mask_(other.bucket_count_mask_)
, bucket_count_(other.bucket_count_)
, begin_bucket_(other.begin_bucket_) {
other.drop();
}
void operator=(FlatHashTable &&other) noexcept {
clear();
nodes_ = other.nodes_;
used_node_count_ = other.used_node_count_;
bucket_count_mask_ = other.bucket_count_mask_;
bucket_count_ = other.bucket_count_;
begin_bucket_ = other.begin_bucket_;
other.drop();
}
~FlatHashTable() {
clear_nodes(nodes_);
}
void swap(FlatHashTable &other) noexcept {
std::swap(nodes_, other.nodes_);
std::swap(used_node_count_, other.used_node_count_);
std::swap(bucket_count_mask_, other.bucket_count_mask_);
std::swap(bucket_count_, other.bucket_count_);
std::swap(begin_bucket_, other.begin_bucket_);
}
uint32 bucket_count() const {
return bucket_count_;
}
NodePointer find(const KeyT &key) {
return NodePointer(find_impl(key));
}
ConstNodePointer find(const KeyT &key) const {
return ConstNodePointer(const_cast<FlatHashTable *>(this)->find_impl(key));
}
size_t size() const {
return used_node_count_;
}
bool empty() const {
return used_node_count_ == 0;
}
Iterator begin() {
return create_iterator(begin_impl());
}
Iterator end() {
return Iterator();
}
ConstIterator begin() const {
return ConstIterator(const_cast<FlatHashTable *>(this)->begin());
}
ConstIterator end() const {
return ConstIterator();
}
void reserve(size_t size) {
if (size == 0) {
return;
}
CHECK(size <= (1u << 29));
uint32 want_size = detail::normalize_flat_hash_table_size(static_cast<uint32>(size) * 5 / 3 + 1);
if (want_size > bucket_count()) {
resize(want_size);
}
}
template <class... ArgsT>
std::pair<NodePointer, bool> emplace(KeyT key, ArgsT &&...args) {
CHECK(!is_hash_table_key_empty<EqT>(key));
if (unlikely(bucket_count_mask_ == 0)) {
CHECK(used_node_count_ == 0);
resize(8);
}
auto bucket = calc_bucket(key);
while (true) {
auto &node = nodes_[bucket];
if (node.empty()) {
if (unlikely(used_node_count_ * 5 >= bucket_count_mask_ * 3)) {
resize(2 * bucket_count_);
CHECK(used_node_count_ * 5 < bucket_count_mask_ * 3);
return emplace(std::move(key), std::forward<ArgsT>(args)...);
}
invalidate_iterators();
node.emplace(std::move(key), std::forward<ArgsT>(args)...);
used_node_count_++;
return {NodePointer(&node), true};
}
if (EqT()(node.key(), key)) {
return {NodePointer(&node), false};
}
next_bucket(bucket);
}
}
std::pair<NodePointer, bool> insert(KeyT key) {
return emplace(std::move(key));
}
template <class ItT>
void insert(ItT begin, ItT end) {
for (; begin != end; ++begin) {
emplace(*begin);
}
}
template <class T = typename NodeT::second_type>
T &operator[](const KeyT &key) {
return emplace(key).first->second;
}
size_t erase(const KeyT &key) {
auto *node = find_impl(key);
if (node == nullptr) {
return 0;
}
erase_node(node);
try_shrink();
return 1;
}
size_t count(const KeyT &key) const {
return const_cast<FlatHashTable *>(this)->find_impl(key) != nullptr;
}
void clear() {
if (nodes_ != nullptr) {
clear_nodes(nodes_);
drop();
}
}
void erase(Iterator it) {
DCHECK(it != end());
erase_node(it.get());
try_shrink();
}
void erase(NodePointer it) {
DCHECK(it != end());
erase_node(it.get());
try_shrink();
}
template <class F>
bool remove_if(F &&f) {
if (empty()) {
return false;
}
auto it = begin_impl();
auto end = nodes_ + bucket_count();
while (it != end && !it->empty()) {
++it;
}
if (it == end) {
do {
--it;
} while (!it->empty());
}
auto first_empty = it;
bool is_removed = false;
while (it != end) {
if (!it->empty() && f(it->get_public())) {
erase_node(it);
is_removed = true;
} else {
++it;
}
}
for (it = nodes_; it != first_empty;) {
if (!it->empty() && f(it->get_public())) {
erase_node(it);
is_removed = true;
} else {
++it;
}
}
try_shrink();
return is_removed;
}
private:
NodeT *nodes_ = nullptr;
uint32 used_node_count_ = 0;
uint32 bucket_count_mask_ = 0;
uint32 bucket_count_ = 0;
uint32 begin_bucket_ = 0;
void drop() {
nodes_ = nullptr;
used_node_count_ = 0;
bucket_count_mask_ = 0;
bucket_count_ = 0;
begin_bucket_ = 0;
}
NodeT *begin_impl() {
if (empty()) {
return nullptr;
}
if (begin_bucket_ == INVALID_BUCKET) {
begin_bucket_ = detail::get_random_flat_hash_table_bucket(bucket_count_mask_);
while (nodes_[begin_bucket_].empty()) {
next_bucket(begin_bucket_);
}
}
return nodes_ + begin_bucket_;
}
NodeT *find_impl(const KeyT &key) {
if (unlikely(nodes_ == nullptr) || is_hash_table_key_empty<EqT>(key)) {
return nullptr;
}
auto bucket = calc_bucket(key);
while (true) {
auto &node = nodes_[bucket];
if (node.empty()) {
return nullptr;
}
if (EqT()(node.key(), key)) {
return &node;
}
next_bucket(bucket);
}
}
void try_shrink() {
DCHECK(nodes_ != nullptr);
if (unlikely(used_node_count_ * 10 < bucket_count_mask_ && bucket_count_mask_ > 7)) {
resize(detail::normalize_flat_hash_table_size((used_node_count_ + 1) * 5 / 3 + 1));
}
invalidate_iterators();
}
uint32 calc_bucket(const KeyT &key) const {
return HashT()(key) & bucket_count_mask_;
}
inline void next_bucket(uint32 &bucket) const {
bucket = (bucket + 1) & bucket_count_mask_;
}
void resize(uint32 new_size) {
if (unlikely(nodes_ == nullptr)) {
allocate_nodes(new_size);
used_node_count_ = 0;
return;
}
auto old_nodes = nodes_;
uint32 old_size = used_node_count_;
uint32 old_bucket_count = bucket_count_;
allocate_nodes(new_size);
used_node_count_ = old_size;
auto old_nodes_end = old_nodes + old_bucket_count;
for (NodeT *old_node = old_nodes; old_node != old_nodes_end; ++old_node) {
if (old_node->empty()) {
continue;
}
auto bucket = calc_bucket(old_node->key());
while (!nodes_[bucket].empty()) {
next_bucket(bucket);
}
nodes_[bucket] = std::move(*old_node);
}
clear_nodes(old_nodes);
}
void erase_node(NodeT *it) {
DCHECK(nodes_ <= it && static_cast<size_t>(it - nodes_) < bucket_count());
it->clear();
used_node_count_--;
const auto bucket_count = bucket_count_;
const auto *end = nodes_ + bucket_count;
for (auto *test_node = it + 1; test_node != end; test_node++) {
if (likely(test_node->empty())) {
return;
}
auto want_node = nodes_ + calc_bucket(test_node->key());
if (want_node <= it || want_node > test_node) {
*it = std::move(*test_node);
it = test_node;
}
}
auto empty_i = static_cast<uint32>(it - nodes_);
auto empty_bucket = empty_i;
for (uint32 test_i = bucket_count;; test_i++) {
auto test_bucket = test_i - bucket_count_;
if (nodes_[test_bucket].empty()) {
return;
}
auto want_i = calc_bucket(nodes_[test_bucket].key());
if (want_i < empty_i) {
want_i += bucket_count;
}
if (want_i <= empty_i || want_i > test_i) {
nodes_[empty_bucket] = std::move(nodes_[test_bucket]);
empty_i = test_i;
empty_bucket = test_bucket;
}
}
}
Iterator create_iterator(NodeT *node) {
return Iterator(node, nodes_, nodes_ + bucket_count());
}
void invalidate_iterators() {
begin_bucket_ = INVALID_BUCKET;
}
};
} // namespace td

87
td/tdutils/td/utils/FloodControlFast.h Normal file
View File

@@ -0,0 +1,87 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
namespace td {
class FloodControlFast {
public:
void add_event(double now) {
for (auto &bucket : buckets_) {
bucket.add_event(now);
wakeup_at_ = td::max(wakeup_at_, bucket.get_wakeup_at());
}
}
double get_wakeup_at() const {
return wakeup_at_;
}
void add_limit(double duration, double count) {
buckets_.emplace_back(duration, count);
}
void clear_events() {
for (auto &bucket : buckets_) {
bucket.clear_events();
}
wakeup_at_ = 0;
}
private:
class FloodControlBucket {
public:
FloodControlBucket(double duration, double count)
: max_capacity_(count - 1), speed_(count / duration), volume_(max_capacity_) {
}
void add_event(double now, double size = 1) {
CHECK(now >= wakeup_at_);
update_volume(now);
if (volume_ >= size) {
volume_ -= size;
return;
}
size -= volume_;
volume_ = 0;
wakeup_at_ = volume_at_ + size / speed_;
volume_at_ = wakeup_at_;
}
double get_wakeup_at() const {
return wakeup_at_;
}
void clear_events() {
volume_ = max_capacity_;
volume_at_ = 0;
wakeup_at_ = 0;
}
private:
const double max_capacity_{1};
const double speed_{1};
double volume_{1};
double volume_at_{0};
double wakeup_at_{0};
void update_volume(double now) {
CHECK(now >= volume_at_);
auto passed = now - volume_at_;
volume_ = td::min(volume_ + passed * speed_, max_capacity_);
volume_at_ = now;
}
};
double wakeup_at_ = 0;
vector<FloodControlBucket> buckets_;
};
} // namespace td

32
td/tdutils/td/utils/FloodControlGlobal.cpp Normal file
View File

@@ -0,0 +1,32 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/FloodControlGlobal.h"
namespace td {
FloodControlGlobal::FloodControlGlobal(uint64 limit) : limit_(limit) {
}
void FloodControlGlobal::finish() {
auto old_value = active_count_.fetch_sub(1, std::memory_order_relaxed);
CHECK(old_value > 0);
}
FloodControlGlobal::Guard FloodControlGlobal::try_start() {
auto old_value = active_count_.fetch_add(1, std::memory_order_relaxed);
if (old_value >= limit_) {
finish();
return nullptr;
}
return Guard(this);
}
void FloodControlGlobal::Finish::operator()(FloodControlGlobal *ctrl) const {
ctrl->finish();
}
} // namespace td

35
td/tdutils/td/utils/FloodControlGlobal.h Normal file
View File

@@ -0,0 +1,35 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <atomic>
#include <memory>
namespace td {
// Restricts the total number of events
class FloodControlGlobal {
public:
explicit FloodControlGlobal(uint64 limit);
struct Finish {
void operator()(FloodControlGlobal *ctrl) const;
};
using Guard = std::unique_ptr<FloodControlGlobal, Finish>;
Guard try_start();
private:
std::atomic<uint64> active_count_{0};
uint64 limit_{0};
void finish();
};
} // namespace td

97
td/tdutils/td/utils/FloodControlStrict.h Normal file
View File

@@ -0,0 +1,97 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <limits>
namespace td {
// More strict implementations of flood control than FloodControlFast.
// Should be just fine for small counters.
class FloodControlStrict {
public:
// there is no reason to return wakeup_at_, because it will be a time before the next allowed event, not current
void add_event(double now) {
events_.push_back(Event{now});
if (without_update_ > 0) {
without_update_--;
} else {
update(now);
}
}
// no more than count in each duration
void add_limit(int32 duration, size_t count) {
limits_.push_back(Limit{duration, count, 0});
without_update_ = 0;
}
double get_wakeup_at() const {
return wakeup_at_;
}
void clear_events() {
events_.clear();
for (auto &limit : limits_) {
limit.pos_ = 0;
}
without_update_ = 0;
wakeup_at_ = 0.0;
}
private:
void update(double now) {
size_t min_pos = events_.size();
without_update_ = std::numeric_limits<size_t>::max();
for (auto &limit : limits_) {
if (limit.count_ < events_.size() - limit.pos_) {
limit.pos_ = events_.size() - limit.count_;
}
// binary-search? :D
auto end_time = now - limit.duration_;
while (limit.pos_ < events_.size() && events_[limit.pos_].timestamp_ < end_time) {
limit.pos_++;
}
if (limit.count_ + limit.pos_ <= events_.size()) {
CHECK(limit.count_ + limit.pos_ == events_.size());
wakeup_at_ = max(wakeup_at_, events_[limit.pos_].timestamp_ + limit.duration_);
without_update_ = 0;
} else {
without_update_ = min(without_update_, limit.count_ + limit.pos_ - events_.size() - 1);
}
min_pos = min(min_pos, limit.pos_);
}
if (min_pos * 2 > events_.size()) {
for (auto &limit : limits_) {
limit.pos_ -= min_pos;
}
events_.erase(events_.begin(), events_.begin() + min_pos);
}
}
double wakeup_at_ = 0.0;
struct Event {
double timestamp_;
};
struct Limit {
int32 duration_;
size_t count_;
size_t pos_;
};
size_t without_update_ = 0;
vector<Event> events_;
vector<Limit> limits_;
};
} // namespace td

208
td/tdutils/td/utils/Gzip.cpp Normal file
View File

@@ -0,0 +1,208 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/Gzip.h"
char disable_linker_warning_about_empty_file_gzip_cpp TD_UNUSED;
#if TD_HAVE_ZLIB
#include "td/utils/SliceBuilder.h"
#include <cstring>
#include <limits>
#include <utility>
#include <zlib.h>
namespace td {
class Gzip::Impl {
public:
z_stream stream_;
// z_stream is not copyable nor movable
Impl() = default;
Impl(const Impl &) = delete;
Impl &operator=(const Impl &) = delete;
Impl(Impl &&) = delete;
Impl &operator=(Impl &&) = delete;
~Impl() = default;
};
Status Gzip::init_encode() {
CHECK(mode_ == Mode::Empty);
init_common();
mode_ = Mode::Encode;
int ret = deflateInit2(&impl_->stream_, 6, Z_DEFLATED, 15, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) {
return Status::Error(PSLICE() << "zlib deflate init failed: " << ret);
}
return Status::OK();
}
Status Gzip::init_decode() {
CHECK(mode_ == Mode::Empty);
init_common();
mode_ = Mode::Decode;
int ret = inflateInit2(&impl_->stream_, MAX_WBITS + 32);
if (ret != Z_OK) {
return Status::Error(PSLICE() << "zlib inflate init failed: " << ret);
}
return Status::OK();
}
void Gzip::set_input(Slice input) {
CHECK(input_size_ == 0);
CHECK(!close_input_flag_);
CHECK(input.size() <= std::numeric_limits<uInt>::max());
CHECK(impl_->stream_.avail_in == 0);
input_size_ = input.size();
impl_->stream_.avail_in = static_cast<uInt>(input.size());
impl_->stream_.next_in = reinterpret_cast<Bytef *>(const_cast<char *>(input.data()));
}
void Gzip::set_output(MutableSlice output) {
CHECK(output_size_ == 0);
CHECK(output.size() <= std::numeric_limits<uInt>::max());
CHECK(impl_->stream_.avail_out == 0);
output_size_ = output.size();
impl_->stream_.avail_out = static_cast<uInt>(output.size());
impl_->stream_.next_out = reinterpret_cast<Bytef *>(output.data());
}
Result<Gzip::State> Gzip::run() {
while (true) {
int ret;
if (mode_ == Mode::Decode) {
ret = inflate(&impl_->stream_, Z_NO_FLUSH);
} else {
ret = deflate(&impl_->stream_, close_input_flag_ ? Z_FINISH : Z_NO_FLUSH);
}
if (ret == Z_OK) {
return State::Running;
}
if (ret == Z_STREAM_END) {
// TODO(now): fail if input is not empty;
clear();
return State::Done;
}
clear();
return Status::Error(PSLICE() << "zlib error " << ret);
}
}
size_t Gzip::left_input() const {
return impl_->stream_.avail_in;
}
size_t Gzip::left_output() const {
return impl_->stream_.avail_out;
}
void Gzip::init_common() {
std::memset(&impl_->stream_, 0, sizeof(impl_->stream_));
impl_->stream_.zalloc = Z_NULL;
impl_->stream_.zfree = Z_NULL;
impl_->stream_.opaque = Z_NULL;
impl_->stream_.avail_in = 0;
impl_->stream_.next_in = nullptr;
impl_->stream_.avail_out = 0;
impl_->stream_.next_out = nullptr;
input_size_ = 0;
output_size_ = 0;
close_input_flag_ = false;
}
void Gzip::clear() {
if (mode_ == Mode::Decode) {
inflateEnd(&impl_->stream_);
} else if (mode_ == Mode::Encode) {
deflateEnd(&impl_->stream_);
}
mode_ = Mode::Empty;
}
Gzip::Gzip() : impl_(make_unique<Impl>()) {
}
Gzip::Gzip(Gzip &&other) noexcept : Gzip() {
swap(other);
}
Gzip &Gzip::operator=(Gzip &&other) noexcept {
CHECK(this != &other);
clear();
swap(other);
return *this;
}
void Gzip::swap(Gzip &other) {
using std::swap;
swap(impl_, other.impl_);
swap(input_size_, other.input_size_);
swap(output_size_, other.output_size_);
swap(close_input_flag_, other.close_input_flag_);
swap(mode_, other.mode_);
}
Gzip::~Gzip() {
clear();
}
BufferSlice gzdecode(Slice s) {
Gzip gzip;
gzip.init_decode().ensure();
ChainBufferWriter message;
gzip.set_input(s);
gzip.close_input();
double k = 2;
gzip.set_output(message.prepare_append(static_cast<size_t>(static_cast<double>(s.size()) * k)));
while (true) {
auto r_state = gzip.run();
if (r_state.is_error()) {
return BufferSlice();
}
auto state = r_state.ok();
if (state == Gzip::State::Done) {
message.confirm_append(gzip.flush_output());
break;
}
if (gzip.need_input()) {
return BufferSlice();
}
if (gzip.need_output()) {
message.confirm_append(gzip.flush_output());
k *= 1.5;
gzip.set_output(message.prepare_append(static_cast<size_t>(static_cast<double>(gzip.left_input()) * k)));
}
}
return message.extract_reader().move_as_buffer_slice();
}
BufferSlice gzencode(Slice s, double max_compression_ratio) {
Gzip gzip;
gzip.init_encode().ensure();
gzip.set_input(s);
gzip.close_input();
auto max_size = static_cast<size_t>(static_cast<double>(s.size()) * max_compression_ratio);
BufferWriter message{max_size};
gzip.set_output(message.prepare_append());
auto r_state = gzip.run();
if (r_state.is_error()) {
return BufferSlice();
}
auto state = r_state.ok();
if (state != Gzip::State::Done) {
return BufferSlice();
}
message.confirm_append(gzip.flush_output());
return message.as_buffer_slice();
}
} // namespace td
#endif

106
td/tdutils/td/utils/Gzip.h Normal file
View File

@@ -0,0 +1,106 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#if TD_HAVE_ZLIB
#include "td/utils/buffer.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
class Gzip {
public:
Gzip();
Gzip(const Gzip &) = delete;
Gzip &operator=(const Gzip &) = delete;
Gzip(Gzip &&other) noexcept;
Gzip &operator=(Gzip &&other) noexcept;
~Gzip();
enum class Mode { Empty, Encode, Decode };
Status init(Mode mode) TD_WARN_UNUSED_RESULT {
if (mode == Mode::Encode) {
return init_encode();
} else if (mode == Mode::Decode) {
return init_decode();
}
clear();
return Status::OK();
}
Status init_encode() TD_WARN_UNUSED_RESULT;
Status init_decode() TD_WARN_UNUSED_RESULT;
void set_input(Slice input);
void set_output(MutableSlice output);
void close_input() {
close_input_flag_ = true;
}
bool need_input() const {
return left_input() == 0;
}
bool need_output() const {
return left_output() == 0;
}
size_t left_input() const;
size_t left_output() const;
size_t used_input() const {
return input_size_ - left_input();
}
size_t used_output() const {
return output_size_ - left_output();
}
size_t flush_input() {
auto res = used_input();
input_size_ = left_input();
return res;
}
size_t flush_output() {
auto res = used_output();
output_size_ = left_output();
return res;
}
enum class State { Running, Done };
Result<State> run() TD_WARN_UNUSED_RESULT;
private:
class Impl;
unique_ptr<Impl> impl_;
size_t input_size_ = 0;
size_t output_size_ = 0;
bool close_input_flag_ = false;
Mode mode_ = Mode::Empty;
void init_common();
void clear();
void swap(Gzip &other);
};
BufferSlice gzdecode(Slice s);
BufferSlice gzencode(Slice s, double max_compression_ratio);
} // namespace td
#endif

64
td/tdutils/td/utils/GzipByteFlow.cpp Normal file
View File

@@ -0,0 +1,64 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/GzipByteFlow.h"
char disable_linker_warning_about_empty_file_gzipbyteflow_cpp TD_UNUSED;
#if TD_HAVE_ZLIB
#include "td/utils/common.h"
#include "td/utils/Status.h"
namespace td {
bool GzipByteFlow::loop() {
if (gzip_.need_input()) {
auto slice = input_->prepare_read();
if (slice.empty()) {
if (!is_input_active_) {
gzip_.close_input();
} else {
return false;
}
} else {
gzip_.set_input(input_->prepare_read());
}
}
if (gzip_.need_output()) {
auto slice = output_.prepare_append();
CHECK(!slice.empty());
gzip_.set_output(slice);
}
auto r_state = gzip_.run();
auto output_size = gzip_.flush_output();
if (output_size) {
if (output_size > max_output_size_ || total_output_size_ > max_output_size_ - output_size) {
finish(Status::Error("Max output size limit exceeded"));
return false;
}
total_output_size_ += output_size;
output_.confirm_append(output_size);
}
auto input_size = gzip_.flush_input();
if (input_size) {
input_->confirm_read(input_size);
}
if (r_state.is_error()) {
finish(r_state.move_as_error());
return false;
}
auto state = r_state.ok();
if (state == Gzip::State::Done) {
consume_input();
return false;
}
return true;
}
} // namespace td
#endif

46
td/tdutils/td/utils/GzipByteFlow.h Normal file
View File

@@ -0,0 +1,46 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/ByteFlow.h"
#include "td/utils/Gzip.h"
#include <limits>
namespace td {
#if TD_HAVE_ZLIB
class GzipByteFlow final : public ByteFlowBase {
public:
GzipByteFlow() = default;
explicit GzipByteFlow(Gzip::Mode mode) {
gzip_.init(mode).ensure();
}
void init_decode() {
gzip_.init_decode().ensure();
}
void init_encode() {
gzip_.init_encode().ensure();
}
void set_max_output_size(size_t max_output_size) {
max_output_size_ = max_output_size;
}
bool loop() final;
private:
Gzip gzip_;
size_t total_output_size_ = 0;
size_t max_output_size_ = std::numeric_limits<size_t>::max();
};
#endif
} // namespace td

70
td/tdutils/td/utils/Hash.h Normal file
View File

@@ -0,0 +1,70 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#if TD_HAVE_ABSL
#include <absl/hash/hash.h>
#endif
#include <utility>
namespace td {
// A simple wrapper for absl::flat_hash_map, std::unordered_map and probably some our implementation of hash map in
// the future
// We will introduce out own Hashing utility like an absl one.
class Hasher {
public:
Hasher() = default;
explicit Hasher(size_t init_value) : hash_(init_value) {
}
std::size_t finalize() const {
return hash_;
}
static Hasher combine(Hasher hasher, size_t value) {
hasher.hash_ ^= value;
return hasher;
}
template <class A, class B>
static Hasher combine(Hasher hasher, const std::pair<A, B> &value) {
hasher = AbslHashValue(std::move(hasher), value.first);
hasher = AbslHashValue(std::move(hasher), value.second);
return hasher;
}
private:
std::size_t hash_{0};
};
template <class IgnoreT>
class TdHash {
public:
template <class T>
std::size_t operator()(const T &value) const noexcept {
return AbslHashValue(Hasher(), value).finalize();
}
};
#if TD_HAVE_ABSL
template <class T>
using AbslHash = absl::Hash<T>;
#else
template <class T>
using AbslHash = TdHash<T>;
#endif
// default hash implementations
template <class H, class T>
decltype(H::combine(std::declval<H>(), std::declval<T>())) AbslHashValue(H hasher, const T &value) {
return H::combine(std::move(hasher), value);
}
} // namespace td

27
td/tdutils/td/utils/HashMap.h Normal file
View File

@@ -0,0 +1,27 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/Hash.h"
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_map.h>
#else
#include <unordered_map>
#endif
namespace td {
#if TD_HAVE_ABSL
template <class Key, class Value, class H = AbslHash<Key>>
using HashMap = absl::flat_hash_map<Key, Value, H>;
#else
template <class Key, class Value, class H = AbslHash<Key>>
using HashMap = std::unordered_map<Key, Value, H>;
#endif
} // namespace td

27
td/tdutils/td/utils/HashSet.h Normal file
View File

@@ -0,0 +1,27 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/Hash.h"
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_set.h>
#else
#include <unordered_set>
#endif
namespace td {
#if TD_HAVE_ABSL
template <class Key, class H = AbslHash<Key>>
using HashSet = absl::flat_hash_set<Key, H>;
#else
template <class Key, class H = AbslHash<Key>>
using HashSet = std::unordered_set<Key, H>;
#endif
} // namespace td

76
td/tdutils/td/utils/HashTableUtils.h Normal file
View File

@@ -0,0 +1,76 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <cstdint>
#include <functional>
namespace td {
template <class EqT, class KeyT>
bool is_hash_table_key_empty(const KeyT &key) {
return EqT()(key, KeyT());
}
inline uint32 randomize_hash(uint32 h) {
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
}
template <class Type>
struct Hash {
uint32 operator()(const Type &value) const;
};
template <class Type>
struct Hash<Type *> {
uint32 operator()(Type *pointer) const {
return Hash<uint64>()(reinterpret_cast<std::uintptr_t>(pointer));
}
};
template <>
inline uint32 Hash<char>::operator()(const char &value) const {
return randomize_hash(static_cast<uint32>(value));
}
template <>
inline uint32 Hash<int32>::operator()(const int32 &value) const {
return randomize_hash(static_cast<uint32>(value));
}
template <>
inline uint32 Hash<uint32>::operator()(const uint32 &value) const {
return randomize_hash(value);
}
template <>
inline uint32 Hash<int64>::operator()(const int64 &value) const {
return randomize_hash(static_cast<uint32>(value + (value >> 32)));
}
template <>
inline uint32 Hash<uint64>::operator()(const uint64 &value) const {
return randomize_hash(static_cast<uint32>(value + (value >> 32)));
}
template <>
inline uint32 Hash<string>::operator()(const string &value) const {
return static_cast<uint32>(std::hash<string>()(value));
}
inline uint32 combine_hashes(uint32 first_hash, uint32 second_hash) {
return first_hash * 2023654985u + second_hash;
}
} // namespace td

141
td/tdutils/td/utils/HazardPointers.h Normal file
View File

@@ -0,0 +1,141 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <array>
#include <atomic>
#include <memory>
namespace td {
template <class T, int MaxPointersN = 1, class Deleter = std::default_delete<T>>
class HazardPointers {
public:
explicit HazardPointers(size_t threads_n) : threads_(threads_n) {
for (auto &data : threads_) {
for (auto &ptr : data.hazard_) {
// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658
#if TD_GCC && GCC_VERSION <= 40902
ptr = nullptr;
#else
std::atomic_init(&ptr, static_cast<T *>(nullptr));
#endif
}
}
}
HazardPointers(const HazardPointers &) = delete;
HazardPointers &operator=(const HazardPointers &) = delete;
HazardPointers(HazardPointers &&) = delete;
HazardPointers &operator=(HazardPointers &&) = delete;
class Holder {
public:
template <class S>
S *protect(std::atomic<S *> &to_protect) {
return do_protect(hazard_ptr_, to_protect);
}
Holder(HazardPointers &hp, size_t thread_id, size_t pos) : Holder(hp.get_hazard_ptr(thread_id, pos)) {
CHECK(hazard_ptr_.load() == 0);
hazard_ptr_.store(reinterpret_cast<T *>(1));
}
Holder(const Holder &) = delete;
Holder &operator=(const Holder &) = delete;
Holder(Holder &&) = delete;
Holder &operator=(Holder &&) = delete;
~Holder() {
clear();
}
void clear() {
hazard_ptr_.store(nullptr, std::memory_order_release);
}
private:
friend class HazardPointers;
explicit Holder(std::atomic<T *> &ptr) : hazard_ptr_(ptr) {
}
std::atomic<T *> &hazard_ptr_;
};
void retire(size_t thread_id, T *ptr = nullptr) {
CHECK(thread_id < threads_.size());
auto &data = threads_[thread_id];
if (ptr) {
data.to_delete_.push_back(std::unique_ptr<T, Deleter>(ptr));
}
for (auto it = data.to_delete_.begin(); it != data.to_delete_.end();) {
if (!is_protected(it->get())) {
it->reset();
it = data.to_delete_.erase(it);
} else {
++it;
}
}
}
// old inteface
T *protect(size_t thread_id, size_t pos, std::atomic<T *> &ptr) {
return do_protect(get_hazard_ptr(thread_id, pos), ptr);
}
void clear(size_t thread_id, size_t pos) {
do_clear(get_hazard_ptr(thread_id, pos));
}
size_t to_delete_size_unsafe() const {
size_t res = 0;
for (auto &thread_data : threads_) {
res += thread_data.to_delete_.size();
}
return res;
}
private:
struct ThreadData {
std::array<std::atomic<T *>, MaxPointersN> hazard_;
char pad[TD_CONCURRENCY_PAD - sizeof(std::array<std::atomic<T *>, MaxPointersN>)];
// stupid gc
std::vector<std::unique_ptr<T, Deleter>> to_delete_;
char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<std::unique_ptr<T, Deleter>>)];
};
std::vector<ThreadData> threads_;
char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<ThreadData>)];
template <class S>
static S *do_protect(std::atomic<T *> &hazard_ptr, std::atomic<S *> &to_protect) {
T *saved = nullptr;
T *to_save;
while ((to_save = to_protect.load()) != saved) {
hazard_ptr.store(to_save);
saved = to_save;
}
return static_cast<S *>(saved);
}
static void do_clear(std::atomic<T *> &hazard_ptr) {
hazard_ptr.store(nullptr, std::memory_order_release);
}
bool is_protected(T *ptr) {
for (auto &thread_data : threads_) {
for (auto &hazard_ptr : thread_data.hazard_) {
if (hazard_ptr.load() == ptr) {
return true;
}
}
}
return false;
}
std::atomic<T *> &get_hazard_ptr(size_t thread_id, size_t pos) {
CHECK(thread_id < threads_.size());
return threads_[thread_id].hazard_[pos];
}
};
} // namespace td

171
td/tdutils/td/utils/Heap.h Normal file
View File

@@ -0,0 +1,171 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
namespace td {
struct HeapNode {
bool in_heap() const {
return pos_ != -1;
}
bool is_top() const {
return pos_ == 0;
}
void remove() {
pos_ = -1;
}
int32 pos_ = -1;
};
template <class KeyT, int K = 4>
class KHeap {
public:
bool empty() const {
return array_.empty();
}
size_t size() const {
return array_.size();
}
KeyT top_key() const {
return array_[0].key_;
}
KeyT get_key(const HeapNode *node) const {
auto pos = static_cast<size_t>(node->pos_);
CHECK(pos < array_.size());
return array_[pos].key_;
}
const HeapNode *top() const {
return array_[0].node_;
}
HeapNode *pop() {
CHECK(!empty());
HeapNode *result = array_[0].node_;
result->remove();
erase(static_cast<size_t>(0));
return result;
}
void insert(KeyT key, HeapNode *node) {
CHECK(!node->in_heap());
array_.push_back({key, node});
fix_up(array_.size() - 1);
}
void fix(KeyT key, HeapNode *node) {
auto pos = static_cast<size_t>(node->pos_);
CHECK(pos < array_.size());
KeyT old_key = array_[pos].key_;
array_[pos].key_ = key;
if (key < old_key) {
fix_up(pos);
} else {
fix_down(pos);
}
}
void erase(HeapNode *node) {
auto pos = static_cast<size_t>(node->pos_);
node->remove();
CHECK(pos < array_.size());
erase(pos);
}
template <class F>
void for_each(F &&f) const {
for (auto &it : array_) {
f(it.key_, it.node_);
}
}
template <class F>
void for_each(F &&f) {
for (auto &it : array_) {
f(it.key_, it.node_);
}
}
void check() const {
for (size_t i = 0; i < array_.size(); i++) {
for (size_t j = i * K + 1; j < i * K + 1 + K && j < array_.size(); j++) {
CHECK(array_[i].key_ <= array_[j].key_);
}
}
}
private:
struct Item {
KeyT key_;
HeapNode *node_;
};
vector<Item> array_;
void fix_up(size_t pos) {
auto item = array_[pos];
while (pos) {
auto parent_pos = (pos - 1) / K;
auto parent_item = array_[parent_pos];
if (parent_item.key_ < item.key_) {
break;
}
parent_item.node_->pos_ = static_cast<int32>(pos);
array_[pos] = parent_item;
pos = parent_pos;
}
item.node_->pos_ = static_cast<int32>(pos);
array_[pos] = item;
}
void fix_down(size_t pos) {
auto item = array_[pos];
while (true) {
auto left_pos = pos * K + 1;
auto right_pos = min(left_pos + K, array_.size());
auto next_pos = pos;
KeyT next_key = item.key_;
for (auto i = left_pos; i < right_pos; i++) {
KeyT i_key = array_[i].key_;
if (i_key < next_key) {
next_key = i_key;
next_pos = i;
}
}
if (next_pos == pos) {
break;
}
array_[pos] = array_[next_pos];
array_[pos].node_->pos_ = static_cast<int32>(pos);
pos = next_pos;
}
item.node_->pos_ = static_cast<int32>(pos);
array_[pos] = item;
}
void erase(size_t pos) {
array_[pos] = array_.back();
array_.pop_back();
if (pos < array_.size()) {
fix_down(pos);
fix_up(pos);
}
if (array_.capacity() > 50 && array_.size() < array_.capacity() / 4) {
array_.shrink_to_fit();
}
}
};
} // namespace td

206
td/tdutils/td/utils/Hints.cpp Normal file
View File

@@ -0,0 +1,206 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/Hints.h"
#include "td/utils/algorithm.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
#include "td/utils/translit.h"
#include "td/utils/utf8.h"
#include <algorithm>
namespace td {
vector<string> Hints::fix_words(vector<string> words) {
std::sort(words.begin(), words.end());
size_t new_words_size = 0;
for (size_t i = 0; i != words.size(); i++) {
if (i == words.size() - 1 || !begins_with(words[i + 1], words[i])) {
if (i != new_words_size) {
words[new_words_size] = std::move(words[i]);
}
new_words_size++;
}
}
if (new_words_size == 1 && words[0].empty()) {
new_words_size = 0;
}
words.resize(new_words_size);
return words;
}
vector<string> Hints::get_words(Slice name) {
return fix_words(utf8_get_search_words(name));
}
void Hints::add_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys) {
vector<KeyT> &keys = word_to_keys[word];
CHECK(!td::contains(keys, key));
keys.push_back(key);
}
void Hints::delete_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys) {
vector<KeyT> &keys = word_to_keys[word];
auto key_it = std::find(keys.begin(), keys.end(), key);
CHECK(key_it != keys.end());
if (keys.size() == 1) {
word_to_keys.erase(word);
} else {
CHECK(keys.size() > 1);
*key_it = keys.back();
keys.pop_back();
}
}
void Hints::add(KeyT key, Slice name) {
// LOG(ERROR) << "Add " << key << ": " << name;
auto it = key_to_name_.find(key);
if (it != key_to_name_.end()) {
if (it->second == name) {
return;
}
vector<string> old_transliterations;
for (auto &old_word : get_words(it->second)) {
delete_word(old_word, key, word_to_keys_);
for (auto &w : get_word_transliterations(old_word, false)) {
if (w != old_word) {
old_transliterations.push_back(std::move(w));
}
}
}
for (auto &word : fix_words(old_transliterations)) {
delete_word(word, key, translit_word_to_keys_);
}
}
if (name.empty()) {
if (it != key_to_name_.end()) {
key_to_name_.erase(it);
}
key_to_rating_.erase(key);
return;
}
vector<string> transliterations;
for (auto &word : get_words(name)) {
add_word(word, key, word_to_keys_);
for (auto &w : get_word_transliterations(word, false)) {
if (w != word) {
transliterations.push_back(std::move(w));
}
}
}
for (auto &word : fix_words(transliterations)) {
add_word(word, key, translit_word_to_keys_);
}
key_to_name_[key] = name.str();
}
void Hints::set_rating(KeyT key, RatingT rating) {
// LOG(ERROR) << "Set rating " << key << ": " << rating;
key_to_rating_[key] = rating;
}
void Hints::add_search_results(vector<KeyT> &results, const string &word,
const std::map<string, vector<KeyT>> &word_to_keys) {
LOG(DEBUG) << "Search for word " << word;
auto it = word_to_keys.lower_bound(word);
while (it != word_to_keys.end() && begins_with(it->first, word)) {
append(results, it->second);
++it;
}
}
vector<Hints::KeyT> Hints::search_word(const string &word) const {
vector<KeyT> results;
add_search_results(results, word, translit_word_to_keys_);
for (const auto &w : get_word_transliterations(word, true)) {
add_search_results(results, w, word_to_keys_);
}
td::unique(results);
return results;
}
std::pair<size_t, vector<Hints::KeyT>> Hints::search(Slice query, int32 limit, bool return_all_for_empty_query) const {
// LOG(ERROR) << "Search " << query;
vector<KeyT> results;
if (limit < 0) {
return {key_to_name_.size(), std::move(results)};
}
auto words = get_words(query);
if (return_all_for_empty_query && words.empty()) {
results.reserve(key_to_name_.size());
for (auto &it : key_to_name_) {
results.push_back(it.first);
}
}
for (size_t i = 0; i < words.size(); i++) {
vector<KeyT> keys = search_word(words[i]);
if (i == 0) {
results = std::move(keys);
continue;
}
// now need to intersect two lists
size_t results_pos = 0;
size_t keys_pos = 0;
size_t new_results_size = 0;
while (results_pos != results.size() && keys_pos != keys.size()) {
if (results[results_pos] < keys[keys_pos]) {
results_pos++;
} else if (results[results_pos] > keys[keys_pos]) {
keys_pos++;
} else {
results[new_results_size++] = results[results_pos];
results_pos++;
keys_pos++;
}
}
results.resize(new_results_size);
}
auto total_size = results.size();
if (total_size < static_cast<size_t>(limit)) {
std::sort(results.begin(), results.end(), CompareByRating(key_to_rating_));
} else {
std::partial_sort(results.begin(), results.begin() + limit, results.end(), CompareByRating(key_to_rating_));
results.resize(limit);
}
return {total_size, std::move(results)};
}
bool Hints::has_key(KeyT key) const {
return key_to_name_.count(key) > 0;
}
string Hints::key_to_string(KeyT key) const {
auto it = key_to_name_.find(key);
if (it == key_to_name_.end()) {
return string();
}
return it->second;
}
std::pair<size_t, vector<Hints::KeyT>> Hints::search_empty(int32 limit) const {
return search(Slice(), limit, true);
}
size_t Hints::size() const {
return key_to_name_.size();
}
} // namespace td

87
td/tdutils/td/utils/Hints.h Normal file
View File

@@ -0,0 +1,87 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/HashTableUtils.h"
#include "td/utils/Slice.h"
#include <map>
#include <unordered_map>
#include <utility>
namespace td {
// TODO template KeyT
class Hints {
using KeyT = int64;
using RatingT = int64;
public:
void add(KeyT key, Slice name);
void remove(KeyT key) {
add(key, "");
}
void set_rating(KeyT key, RatingT rating);
std::pair<size_t, vector<KeyT>> search(
Slice query, int32 limit,
bool return_all_for_empty_query = false) const; // TODO sort by name instead of sort by rating
bool has_key(KeyT key) const;
string key_to_string(KeyT key) const;
std::pair<size_t, vector<KeyT>> search_empty(int32 limit) const; // == search("", limit, true)
size_t size() const;
static vector<string> fix_words(vector<string> words);
private:
std::map<string, vector<KeyT>> word_to_keys_;
std::map<string, vector<KeyT>> translit_word_to_keys_;
std::unordered_map<KeyT, string, Hash<KeyT>> key_to_name_;
std::unordered_map<KeyT, RatingT, Hash<KeyT>> key_to_rating_;
static void add_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys);
static void delete_word(const string &word, KeyT key, std::map<string, vector<KeyT>> &word_to_keys);
static vector<string> get_words(Slice name);
static void add_search_results(vector<KeyT> &results, const string &word,
const std::map<string, vector<KeyT>> &word_to_keys);
vector<KeyT> search_word(const string &word) const;
class CompareByRating {
const std::unordered_map<KeyT, RatingT, Hash<KeyT>> &key_to_rating_;
RatingT get_rating(const KeyT &key) const {
auto it = key_to_rating_.find(key);
if (it == key_to_rating_.end()) {
return RatingT();
}
return it->second;
}
public:
explicit CompareByRating(const std::unordered_map<KeyT, RatingT, Hash<KeyT>> &key_to_rating)
: key_to_rating_(key_to_rating) {
}
bool operator()(const KeyT &lhs, const KeyT &rhs) const {
auto lhs_rating = get_rating(lhs);
auto rhs_rating = get_rating(rhs);
return lhs_rating < rhs_rating || (lhs_rating == rhs_rating && lhs < rhs);
}
};
};
} // namespace td

92
td/tdutils/td/utils/HttpDate.cpp Normal file
View File

@@ -0,0 +1,92 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/HttpDate.h"
#include "td/utils/misc.h"
#include "td/utils/Parser.h"
#include "td/utils/Slice.h"
namespace td {
Result<int32> HttpDate::to_unix_time(int32 year, int32 month, int32 day, int32 hour, int32 minute, int32 second) {
if (year < 1970 || year > 2037) {
return Status::Error("Invalid year");
}
if (month < 1 || month > 12) {
return Status::Error("Invalid month");
}
if (day < 1 || day > days_in_month(year, month)) {
return Status::Error("Invalid day");
}
if (hour < 0 || hour >= 24) {
return Status::Error("Invalid hour");
}
if (minute < 0 || minute >= 60) {
return Status::Error("Invalid minute");
}
if (second < 0 || second > 60) {
return Status::Error("Invalid second");
}
int32 res = 0;
for (int32 y = 1970; y < year; y++) {
res += (is_leap(y) + 365) * seconds_in_day();
}
for (int32 m = 1; m < month; m++) {
res += days_in_month(year, m) * seconds_in_day();
}
res += (day - 1) * seconds_in_day();
res += hour * 60 * 60;
res += minute * 60;
res += second;
return res;
}
Result<int32> HttpDate::parse_http_date(string slice) {
Parser p(slice);
p.read_till(','); // ignore week day
p.skip(',');
p.skip_whitespaces();
p.skip_nofail('0');
TRY_RESULT(day, to_integer_safe<int32>(p.read_word()));
auto month_name = p.read_word();
to_lower_inplace(month_name);
TRY_RESULT(year, to_integer_safe<int32>(p.read_word()));
p.skip_whitespaces();
p.skip_nofail('0');
TRY_RESULT(hour, to_integer_safe<int32>(p.read_till(':')));
p.skip(':');
p.skip_nofail('0');
TRY_RESULT(minute, to_integer_safe<int32>(p.read_till(':')));
p.skip(':');
p.skip_nofail('0');
TRY_RESULT(second, to_integer_safe<int32>(p.read_word()));
auto gmt = p.read_word();
TRY_STATUS(std::move(p.status()));
if (gmt != "GMT") {
return Status::Error("Timezone must be GMT");
}
static Slice month_names[12] = {"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"};
int month = 0;
for (int m = 1; m <= 12; m++) {
if (month_names[m - 1] == month_name) {
month = m;
break;
}
}
if (month == 0) {
return Status::Error("Unknown month name");
}
return to_unix_time(year, month, day, hour, minute, second);
}
} // namespace td

34
td/tdutils/td/utils/HttpDate.h Normal file
View File

@@ -0,0 +1,34 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Status.h"
namespace td {
class HttpDate {
static bool is_leap(int32 year) {
return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
}
static int32 seconds_in_day() {
return 24 * 60 * 60;
}
public:
static int32 days_in_month(int32 year, int32 month) {
static int cnt[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
return cnt[month - 1] + (month == 2 && is_leap(year));
}
static Result<int32> to_unix_time(int32 year, int32 month, int32 day, int32 hour, int32 minute, int32 second);
static Result<int32> parse_http_date(string slice);
};
} // namespace td

272
td/tdutils/td/utils/HttpUrl.cpp Normal file
View File

@@ -0,0 +1,272 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/HttpUrl.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Parser.h"
#include "td/utils/port/IPAddress.h"
#include "td/utils/SliceBuilder.h"
#include <algorithm>
namespace td {
string HttpUrl::get_url() const {
string result;
switch (protocol_) {
case Protocol::Http:
result += "http://";
break;
case Protocol::Https:
result += "https://";
break;
default:
UNREACHABLE();
}
if (!userinfo_.empty()) {
result += userinfo_;
result += '@';
}
result += host_;
if (specified_port_ > 0) {
result += ':';
result += to_string(specified_port_);
}
LOG_CHECK(!query_.empty() && query_[0] == '/') << query_;
result += query_;
return result;
}
Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
// url == [https?://][userinfo@]host[:port]
ConstParser parser(url);
string protocol_str = to_lower(parser.read_till_nofail(":/?#@[]"));
HttpUrl::Protocol protocol;
if (parser.try_skip("://")) {
if (protocol_str == "http") {
protocol = HttpUrl::Protocol::Http;
} else if (protocol_str == "https") {
protocol = HttpUrl::Protocol::Https;
} else {
return Status::Error("Unsupported URL protocol");
}
} else {
parser = ConstParser(url);
protocol = default_protocol;
}
Slice userinfo_host_port = parser.read_till_nofail("/?#");
int port = 0;
const char *colon = userinfo_host_port.end() - 1;
while (colon > userinfo_host_port.begin() && *colon != ':' && *colon != ']' && *colon != '@') {
colon--;
}
Slice userinfo_host;
if (colon > userinfo_host_port.begin() && *colon == ':') {
Slice port_slice(colon + 1, userinfo_host_port.end());
while (port_slice.size() > 1 && port_slice[0] == '0') {
port_slice.remove_prefix(1);
}
auto r_port = to_integer_safe<int>(port_slice);
if (r_port.is_error() || r_port.ok() == 0) {
port = -1;
} else {
port = r_port.ok();
}
userinfo_host = Slice(userinfo_host_port.begin(), colon);
} else {
userinfo_host = userinfo_host_port;
}
if (port < 0 || port > 65535) {
return Status::Error("Wrong port number specified in the URL");
}
auto at_pos = userinfo_host.rfind('@');
Slice userinfo = at_pos == static_cast<size_t>(-1) ? "" : userinfo_host.substr(0, at_pos);
Slice host = userinfo_host.substr(at_pos + 1);
bool is_ipv6 = false;
if (!host.empty() && host[0] == '[' && host.back() == ']') {
IPAddress ip_address;
if (ip_address.init_ipv6_port(host.str(), 1).is_error()) {
return Status::Error("Wrong IPv6 address specified in the URL");
}
CHECK(ip_address.is_ipv6());
is_ipv6 = true;
}
if (host.empty()) {
return Status::Error("URL host is empty");
}
if (host == ".") {
return Status::Error("Host is invalid");
}
int specified_port = port;
if (port == 0) {
if (protocol == HttpUrl::Protocol::Http) {
port = 80;
} else {
CHECK(protocol == HttpUrl::Protocol::Https);
port = 443;
}
}
Slice query = parser.read_all();
while (!query.empty() && is_space(query.back())) {
query.remove_suffix(1);
}
if (query.empty()) {
query = Slice("/");
}
string query_str;
if (query[0] != '/') {
query_str = '/';
}
for (auto c : query) {
if (static_cast<unsigned char>(c) <= 0x20) {
query_str += '%';
query_str += "0123456789ABCDEF"[c / 16];
query_str += "0123456789ABCDEF"[c % 16];
} else {
query_str += c;
}
}
auto check_url_part = [](Slice part, Slice name, bool allow_colon) {
for (size_t i = 0; i < part.size(); i++) {
char c = part[i];
if (is_alnum(c) || c == '.' || c == '-' || c == '_' || c == '!' || c == '$' || c == ',' || c == '~' || c == '*' ||
c == '\'' || c == '(' || c == ')' || c == ';' || c == '&' || c == '+' || c == '=' ||
(allow_colon && c == ':')) {
// symbols allowed by RFC 7230 and RFC 3986
continue;
}
if (c == '%') {
c = part[++i];
if (is_hex_digit(c)) {
c = part[++i];
if (is_hex_digit(c)) {
// percent encoded symbol as allowed by RFC 7230 and RFC 3986
continue;
}
}
return Status::Error(PSLICE() << "Wrong percent-encoded symbol in URL " << name);
}
// all other symbols aren't allowed
auto uc = static_cast<unsigned char>(c);
if (uc >= 128) {
// but we allow plain UTF-8 symbols
continue;
}
return Status::Error(PSLICE() << "Disallowed character in URL " << name);
}
return Status::OK();
};
string host_str = to_lower(host);
if (is_ipv6) {
for (size_t i = 1; i + 1 < host_str.size(); i++) {
char c = host_str[i];
if (c == ':' || ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || c == '.') {
continue;
}
return Status::Error("Wrong IPv6 URL host");
}
} else {
TRY_STATUS(check_url_part(host_str, "host", false));
TRY_STATUS(check_url_part(userinfo, "userinfo", true));
}
return HttpUrl{protocol, userinfo.str(), std::move(host_str), is_ipv6, specified_port, port, std::move(query_str)};
}
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) {
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::Http ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
<< tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_);
return sb;
}
HttpUrlQuery parse_url_query(Slice query) {
if (!query.empty() && query[0] == '/') {
query.remove_prefix(1);
}
size_t path_size = 0;
while (path_size < query.size() && query[path_size] != '?' && query[path_size] != '#') {
path_size++;
}
HttpUrlQuery result;
result.path_ = full_split(url_decode(query.substr(0, path_size), false), '/');
while (!result.path_.empty() && result.path_.back().empty()) {
result.path_.pop_back();
}
if (path_size < query.size() && query[path_size] == '?') {
query = query.substr(path_size + 1);
query.truncate(query.find('#'));
ConstParser parser(query);
while (!parser.data().empty()) {
auto key_value = split(parser.read_till_nofail('&'), '=');
parser.skip_nofail('&');
auto key = url_decode(key_value.first, true);
if (!key.empty()) {
result.args_.emplace_back(std::move(key), url_decode(key_value.second, true));
}
}
CHECK(parser.status().is_ok());
}
return result;
}
bool HttpUrlQuery::has_arg(Slice key) const {
auto it =
std::find_if(args_.begin(), args_.end(), [&key](const std::pair<string, string> &s) { return s.first == key; });
return it != args_.end();
}
Slice HttpUrlQuery::get_arg(Slice key) const {
auto it =
std::find_if(args_.begin(), args_.end(), [&key](const std::pair<string, string> &s) { return s.first == key; });
return it == args_.end() ? Slice() : it->second;
}
string get_url_host(Slice url) {
auto r_http_url = parse_url(url);
if (r_http_url.is_error()) {
return string();
}
return r_http_url.ok().host_;
}
string get_url_query_file_name(const string &query) {
Slice query_slice = query;
query_slice.truncate(query.find_first_of("?#"));
auto slash_pos = query_slice.rfind('/');
if (slash_pos < query_slice.size()) {
return query_slice.substr(slash_pos + 1).str();
}
return query_slice.str();
}
string get_url_file_name(Slice url) {
auto r_http_url = parse_url(url);
if (r_http_url.is_error()) {
LOG(WARNING) << "Receive wrong URL \"" << url << '"';
return string();
}
return get_url_query_file_name(r_http_url.ok().query_);
}
} // namespace td

64
td/tdutils/td/utils/HttpUrl.h Normal file
View File

@@ -0,0 +1,64 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
#include <utility>
namespace td {
class HttpUrl {
public:
enum class Protocol { Http, Https } protocol_ = Protocol::Http;
string userinfo_;
string host_;
bool is_ipv6_ = false;
int specified_port_ = 0;
int port_ = 0;
string query_;
string get_url() const;
HttpUrl(Protocol protocol, string userinfo, string host, bool is_ipv6, int specified_port, int port, string query)
: protocol_(protocol)
, userinfo_(std::move(userinfo))
, host_(std::move(host))
, is_ipv6_(is_ipv6)
, specified_port_(specified_port)
, port_(port)
, query_(std::move(query)) {
}
};
Result<HttpUrl> parse_url(Slice url,
HttpUrl::Protocol default_protocol = HttpUrl::Protocol::Http) TD_WARN_UNUSED_RESULT;
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url);
class HttpUrlQuery {
public:
vector<string> path_;
vector<std::pair<string, string>> args_;
bool has_arg(Slice key) const;
Slice get_arg(Slice key) const;
};
HttpUrlQuery parse_url_query(Slice query);
string get_url_host(Slice url);
string get_url_query_file_name(const string &query);
string get_url_file_name(Slice url);
} // namespace td

745
td/tdutils/td/utils/JsonBuilder.cpp Normal file
View File

@@ -0,0 +1,745 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/JsonBuilder.h"
#include "td/utils/misc.h"
#include "td/utils/ScopeGuard.h"
#include "td/utils/SliceBuilder.h"
#include "td/utils/utf8.h"
namespace td {
StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val) {
sb << '"';
SCOPE_EXIT {
sb << '"';
};
auto *s = val.value_.begin();
auto len = val.value_.size();
for (size_t pos = 0; pos < len; pos++) {
auto ch = static_cast<unsigned char>(s[pos]);
switch (ch) {
case '"':
sb << '\\' << '"';
break;
case '\\':
sb << '\\' << '\\';
break;
case '\b':
sb << '\\' << 'b';
break;
case '\f':
sb << '\\' << 'f';
break;
case '\n':
sb << '\\' << 'n';
break;
case '\r':
sb << '\\' << 'r';
break;
case '\t':
sb << '\\' << 't';
break;
default:
if (ch <= 31) {
sb << JsonOneChar(s[pos]);
break;
}
sb << s[pos];
break;
}
}
return sb;
}
StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) {
sb << '"';
SCOPE_EXIT {
sb << '"';
};
auto *s = val.str_.begin();
auto len = val.str_.size();
for (size_t pos = 0; pos < len; pos++) {
auto ch = static_cast<unsigned char>(s[pos]);
switch (ch) {
case '"':
sb << '\\' << '"';
break;
case '\\':
sb << '\\' << '\\';
break;
case '\b':
sb << '\\' << 'b';
break;
case '\f':
sb << '\\' << 'f';
break;
case '\n':
sb << '\\' << 'n';
break;
case '\r':
sb << '\\' << 'r';
break;
case '\t':
sb << '\\' << 't';
break;
default:
if (ch <= 31) {
sb << JsonOneChar(s[pos]);
break;
}
if (128 <= ch) {
uint32 a = ch;
CHECK((a & 0x40) != 0);
CHECK(pos + 1 < len);
uint32 b = static_cast<unsigned char>(s[++pos]);
CHECK((b & 0xc0) == 0x80);
if ((a & 0x20) == 0) {
CHECK((a & 0x1e) > 0);
sb << JsonChar(((a & 0x1f) << 6) | (b & 0x3f));
break;
}
CHECK(pos + 1 < len);
uint32 c = static_cast<unsigned char>(s[++pos]);
CHECK((c & 0xc0) == 0x80);
if ((a & 0x10) == 0) {
CHECK(((a & 0x0f) | (b & 0x20)) > 0);
sb << JsonChar(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f));
break;
}
CHECK(pos + 1 < len);
uint32 d = static_cast<unsigned char>(s[++pos]);
CHECK((d & 0xc0) == 0x80);
if ((a & 0x08) == 0) {
CHECK(((a & 0x07) | (b & 0x30)) > 0);
sb << JsonChar(((a & 0x07) << 18) | ((b & 0x3f) << 12) | ((c & 0x3f) << 6) | (d & 0x3f));
break;
}
UNREACHABLE();
break;
}
sb << s[pos];
break;
}
}
return sb;
}
Result<MutableSlice> json_string_decode(Parser &parser) {
if (!parser.try_skip('"')) {
return Status::Error("Opening '\"' expected");
}
auto data = parser.data();
auto *result_start = data.ubegin();
auto *cur_src = result_start;
auto *cur_dest = result_start;
auto *end = data.uend();
while (true) {
if (cur_src == end) {
return Status::Error("Closing '\"' not found");
}
if (*cur_src == '"') {
parser.advance(cur_src + 1 - result_start);
return data.substr(0, cur_dest - result_start);
}
if (*cur_src == '\\') {
cur_src++;
if (cur_src == end) {
return Status::Error("Closing '\"' not found");
}
switch (*cur_src) {
case 'b':
*cur_dest++ = '\b';
cur_src++;
break;
case 'f':
*cur_dest++ = '\f';
cur_src++;
break;
case 'n':
*cur_dest++ = '\n';
cur_src++;
break;
case 'r':
*cur_dest++ = '\r';
cur_src++;
break;
case 't':
*cur_dest++ = '\t';
cur_src++;
break;
case 'u': {
cur_src++;
if (cur_src + 4 > end) {
return Status::Error("\\u has less than 4 symbols");
}
uint32 num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
num = num * 16 + d;
}
if (0xD7FF < num && num < 0xE000) {
if (cur_src + 6 <= end && cur_src[0] == '\\' && cur_src[1] == 'u') {
cur_src += 2;
int new_num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
new_num = new_num * 16 + d;
}
if (0xD7FF < new_num && new_num < 0xE000) {
num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000;
} else {
cur_src -= 6;
}
}
}
cur_dest = append_utf8_character_unsafe(cur_dest, num);
break;
}
default:
*cur_dest++ = *cur_src++;
break;
}
} else {
*cur_dest++ = *cur_src++;
}
}
UNREACHABLE();
return {};
}
Status json_string_skip(Parser &parser) {
if (!parser.try_skip('"')) {
return Status::Error("Opening '\"' expected");
}
auto data = parser.data();
auto *cur_src = data.ubegin();
auto *end = data.uend();
while (true) {
if (cur_src == end) {
return Status::Error("Closing '\"' not found");
}
if (*cur_src == '"') {
parser.advance(cur_src + 1 - data.ubegin());
return Status::OK();
}
if (*cur_src == '\\') {
cur_src++;
if (cur_src == end) {
return Status::Error("Closing '\"' not found");
}
switch (*cur_src) {
case 'u': {
cur_src++;
if (cur_src + 4 > end) {
return Status::Error("\\u has less than 4 symbols");
}
int num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
num = num * 16 + d;
}
if (0xD7FF < num && num < 0xE000) {
if (cur_src + 6 <= end && cur_src[0] == '\\' && cur_src[1] == 'u') {
cur_src += 2;
int new_num = 0;
for (int i = 0; i < 4; i++, cur_src++) {
int d = hex_to_int(*cur_src);
if (d == 16) {
return Status::Error("Invalid \\u -- not hex digit");
}
new_num = new_num * 16 + d;
}
if (0xD7FF < new_num && new_num < 0xE000) {
// num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000;
} else {
cur_src -= 6;
}
}
}
break;
}
default:
cur_src++;
break;
}
} else {
cur_src++;
}
}
UNREACHABLE();
return Status::OK();
}
Result<JsonValue> do_json_decode(Parser &parser, int32 max_depth) {
if (max_depth < 0) {
return Status::Error("Too big object depth");
}
parser.skip_whitespaces();
switch (parser.peek_char()) {
case 'f':
if (parser.try_skip("false")) {
return JsonValue::create_boolean(false);
}
return Status::Error("Token starts with 'f' -- false expected");
case 't':
if (parser.try_skip("true")) {
return JsonValue::create_boolean(true);
}
return Status::Error("Token starts with 't' -- true expected");
case 'n':
if (parser.try_skip("null")) {
return JsonValue();
}
return Status::Error("Token starts with 'n' -- null expected");
case '"': {
TRY_RESULT(slice, json_string_decode(parser));
return JsonValue::create_string(slice);
}
case '[': {
parser.skip('[');
parser.skip_whitespaces();
vector<JsonValue> res;
if (parser.try_skip(']')) {
return JsonValue::create_array(std::move(res));
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
TRY_RESULT(value, do_json_decode(parser, max_depth - 1));
res.emplace_back(std::move(value));
parser.skip_whitespaces();
if (parser.try_skip(']')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
return Status::Error("Unexpected symbol while parsing JSON Array");
}
return JsonValue::create_array(std::move(res));
}
case '{': {
parser.skip('{');
parser.skip_whitespaces();
if (parser.try_skip('}')) {
return JsonValue::make_object(JsonObject());
}
vector<std::pair<Slice, JsonValue>> field_values;
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
TRY_RESULT(field, json_string_decode(parser));
parser.skip_whitespaces();
if (!parser.try_skip(':')) {
return Status::Error("':' expected");
}
TRY_RESULT(value, do_json_decode(parser, max_depth - 1));
field_values.emplace_back(field, std::move(value));
parser.skip_whitespaces();
if (parser.try_skip('}')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
if (parser.empty()) {
return Status::Error("Unexpected string end");
}
return Status::Error("Unexpected symbol while parsing JSON Object");
}
return JsonValue::make_object(JsonObject(std::move(field_values)));
}
case '-':
case '+':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
auto num = parser.read_while(
[](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; });
return JsonValue::create_number(num);
}
case 0:
return Status::Error("Unexpected string end");
default: {
char next = parser.peek_char();
if (0 < next && next < 127) {
return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'");
} else {
return Status::Error("Unexpected symbol");
}
}
}
UNREACHABLE();
}
Status do_json_skip(Parser &parser, int32 max_depth) {
if (max_depth < 0) {
return Status::Error("Too big object depth");
}
parser.skip_whitespaces();
switch (parser.peek_char()) {
case 'f':
if (parser.try_skip("false")) {
return Status::OK();
}
return Status::Error("Starts with 'f' -- false expected");
case 't':
if (parser.try_skip("true")) {
return Status::OK();
}
return Status::Error("Starts with 't' -- true expected");
case 'n':
if (parser.try_skip("null")) {
return Status::OK();
}
return Status::Error("Starts with 'n' -- null expected");
case '"': {
return json_string_skip(parser);
}
case '[': {
parser.skip('[');
parser.skip_whitespaces();
if (parser.try_skip(']')) {
return Status::OK();
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected end");
}
TRY_STATUS(do_json_skip(parser, max_depth - 1));
parser.skip_whitespaces();
if (parser.try_skip(']')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
return Status::Error("Unexpected symbol");
}
return Status::OK();
}
case '{': {
parser.skip('{');
parser.skip_whitespaces();
if (parser.try_skip('}')) {
return Status::OK();
}
while (true) {
if (parser.empty()) {
return Status::Error("Unexpected end");
}
TRY_STATUS(json_string_skip(parser));
parser.skip_whitespaces();
if (!parser.try_skip(':')) {
return Status::Error("':' expected");
}
TRY_STATUS(do_json_skip(parser, max_depth - 1));
parser.skip_whitespaces();
if (parser.try_skip('}')) {
break;
}
if (parser.try_skip(',')) {
parser.skip_whitespaces();
continue;
}
return Status::Error("Unexpected symbol");
}
return Status::OK();
}
case '-':
case '+':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
parser.read_while(
[](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; });
return Status::OK();
}
case 0:
return Status::Error("Unexpected end");
default: {
char next = parser.peek_char();
if (0 < next && next < 127) {
return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'");
} else {
return Status::Error("Unexpected symbol");
}
}
}
return Status::Error("Can't parse");
}
Slice JsonValue::get_type_name(Type type) {
switch (type) {
case Type::Null:
return Slice("Null");
case Type::Number:
return Slice("Number");
case Type::Boolean:
return Slice("Boolean");
case Type::String:
return Slice("String");
case Type::Array:
return Slice("Array");
case Type::Object:
return Slice("Object");
default:
UNREACHABLE();
return Slice("Unknown");
}
}
JsonObject::JsonObject(vector<std::pair<Slice, JsonValue>> &&field_values) : field_values_(std::move(field_values)) {
}
size_t JsonObject::field_count() const {
return field_values_.size();
}
JsonValue JsonObject::extract_field(Slice name) {
for (auto &field_value : field_values_) {
if (field_value.first == name) {
return std::move(field_value.second);
}
}
return JsonValue();
}
Result<JsonValue> JsonObject::extract_optional_field(Slice name, JsonValueType type) {
for (auto &field_value : field_values_) {
if (field_value.first == name) {
if (type != JsonValue::Type::Null && field_value.second.type() != type) {
return Status::Error(400, PSLICE()
<< "Field \"" << name << "\" must be of type " << JsonValue::get_type_name(type));
}
return std::move(field_value.second);
}
}
return JsonValue();
}
Result<JsonValue> JsonObject::extract_required_field(Slice name, JsonValueType type) {
for (auto &field_value : field_values_) {
if (field_value.first == name) {
if (type != JsonValue::Type::Null && field_value.second.type() != type) {
return Status::Error(400, PSLICE()
<< "Field \"" << name << "\" must be of type " << JsonValue::get_type_name(type));
}
return std::move(field_value.second);
}
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\"");
}
const JsonValue *JsonObject::get_field(Slice name) const {
for (auto &field_value : field_values_) {
if (field_value.first == name) {
return &field_value.second;
}
}
return nullptr;
}
bool JsonObject::has_field(Slice name) const {
return get_field(name) != nullptr;
}
Result<bool> JsonObject::get_optional_bool_field(Slice name, bool default_value) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::Boolean) {
return value->get_boolean();
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type Boolean");
}
return default_value;
}
Result<bool> JsonObject::get_required_bool_field(Slice name) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::Boolean) {
return value->get_boolean();
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type Boolean");
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << '"');
}
template <class T>
static Result<T> get_integer_field(Slice name, Slice value) {
auto r_int = to_integer_safe<T>(value);
if (r_int.is_ok()) {
return r_int.ok();
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a valid Number");
}
Result<int32> JsonObject::get_optional_int_field(Slice name, int32 default_value) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::String) {
return get_integer_field<int32>(name, value->get_string());
}
if (value->type() == JsonValue::Type::Number) {
return get_integer_field<int32>(name, value->get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a Number");
}
return default_value;
}
Result<int32> JsonObject::get_required_int_field(Slice name) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::String) {
return get_integer_field<int32>(name, value->get_string());
}
if (value->type() == JsonValue::Type::Number) {
return get_integer_field<int32>(name, value->get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a Number");
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << '"');
}
Result<int64> JsonObject::get_optional_long_field(Slice name, int64 default_value) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::String) {
return get_integer_field<int64>(name, value->get_string());
}
if (value->type() == JsonValue::Type::Number) {
return get_integer_field<int64>(name, value->get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a Number");
}
return default_value;
}
Result<int64> JsonObject::get_required_long_field(Slice name) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::String) {
return get_integer_field<int64>(name, value->get_string());
}
if (value->type() == JsonValue::Type::Number) {
return get_integer_field<int64>(name, value->get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a Number");
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << '"');
}
Result<double> JsonObject::get_optional_double_field(Slice name, double default_value) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::Number) {
return to_double(value->get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type Number");
}
return default_value;
}
Result<double> JsonObject::get_required_double_field(Slice name) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::Number) {
return to_double(value->get_number());
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type Number");
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << '"');
}
Result<string> JsonObject::get_optional_string_field(Slice name, string default_value) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::String) {
return value->get_string().str();
}
if (value->type() == JsonValue::Type::Number) {
return value->get_number().str();
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type String");
}
return std::move(default_value);
}
Result<string> JsonObject::get_required_string_field(Slice name) const {
auto value = get_field(name);
if (value != nullptr) {
if (value->type() == JsonValue::Type::String) {
return value->get_string().str();
}
if (value->type() == JsonValue::Type::Number) {
return value->get_number().str();
}
return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type String");
}
return Status::Error(400, PSLICE() << "Can't find field \"" << name << '"');
}
void JsonObject::foreach(const std::function<void(Slice name, const JsonValue &value)> &callback) const {
for (auto &field_value : field_values_) {
callback(field_value.first, field_value.second);
}
}
} // namespace td

895
td/tdutils/td/utils/JsonBuilder.h Normal file
View File

@@ -0,0 +1,895 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/Parser.h"
#include "td/utils/Slice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
#include <functional>
#include <new>
#include <type_traits>
#include <utility>
namespace td {
class JsonTrue {
public:
friend StringBuilder &operator<<(StringBuilder &sb, const JsonTrue &val) {
return sb << "true";
}
};
class JsonFalse {
public:
friend StringBuilder &operator<<(StringBuilder &sb, const JsonFalse &val) {
return sb << "false";
}
};
class JsonNull {
public:
friend StringBuilder &operator<<(StringBuilder &sb, JsonNull val) {
return sb << "null";
}
};
class JsonBool {
public:
explicit JsonBool(bool value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonBool &val) {
if (val.value_) {
return sb << JsonTrue();
} else {
return sb << JsonFalse();
}
}
private:
bool value_;
};
class JsonInt {
public:
explicit JsonInt(int32 value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonInt &val) {
return sb << val.value_;
}
private:
int32 value_;
};
class JsonLong {
public:
explicit JsonLong(int64 value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonLong &val) {
return sb << val.value_;
}
private:
int64 value_;
};
class JsonFloat {
public:
explicit JsonFloat(double value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonFloat &val) {
return sb << val.value_;
}
private:
double value_;
};
class JsonOneChar {
public:
explicit JsonOneChar(uint32 c) : c_(c) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonOneChar &val) {
auto c = val.c_;
return sb << '\\' << 'u' << "0123456789abcdef"[c >> 12] << "0123456789abcdef"[(c >> 8) & 15]
<< "0123456789abcdef"[(c >> 4) & 15] << "0123456789abcdef"[c & 15];
}
private:
uint32 c_;
};
class JsonChar {
public:
explicit JsonChar(uint32 c) : c_(c) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonChar &val) {
auto c = val.c_;
if (c < 0x10000) {
if (0xD7FF < c && c < 0xE000) {
// UTF-8 correctness has already been checked
UNREACHABLE();
}
return sb << JsonOneChar(c);
} else if (c <= 0x10ffff) {
return sb << JsonOneChar(0xD7C0 + (c >> 10)) << JsonOneChar(0xDC00 + (c & 0x3FF));
} else {
// UTF-8 correctness has already been checked
UNREACHABLE();
}
}
private:
uint32 c_;
};
class JsonRaw {
public:
explicit JsonRaw(Slice value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonRaw &val) {
return sb << val.value_;
}
private:
Slice value_;
};
class JsonRawString {
public:
explicit JsonRawString(Slice value) : value_(value) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val);
private:
Slice value_;
};
class JsonString {
public:
explicit JsonString(Slice str) : str_(str) {
}
friend StringBuilder &operator<<(StringBuilder &sb, const JsonString &val);
private:
Slice str_;
};
class JsonScope;
class JsonValueScope;
class JsonArrayScope;
class JsonObjectScope;
class JsonBuilder {
public:
explicit JsonBuilder(StringBuilder &&sb = {}, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) {
}
StringBuilder &string_builder() {
return sb_;
}
friend class JsonScope;
JsonValueScope enter_value() TD_WARN_UNUSED_RESULT;
JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT;
JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT;
int32 offset() const {
return offset_;
}
bool is_pretty() const {
return offset_ >= 0;
}
void print_offset() {
if (offset_ >= 0) {
sb_ << '\n';
for (int x = 0; x < offset_; x++) {
sb_ << " ";
}
}
}
void dec_offset() {
if (offset_ >= 0) {
CHECK(offset_ > 0);
offset_--;
}
}
void inc_offset() {
if (offset_ >= 0) {
offset_++;
}
}
private:
StringBuilder sb_;
JsonScope *scope_ = nullptr;
int32 offset_;
};
class Jsonable {};
class JsonScope {
public:
explicit JsonScope(JsonBuilder *jb) : sb_(&jb->sb_), jb_(jb), save_scope_(jb->scope_) {
jb_->scope_ = this;
CHECK(is_active());
}
JsonScope(const JsonScope &) = delete;
JsonScope &operator=(const JsonScope &) = delete;
JsonScope(JsonScope &&other) noexcept : sb_(other.sb_), jb_(other.jb_), save_scope_(other.save_scope_) {
other.jb_ = nullptr;
}
JsonScope &operator=(JsonScope &&) = delete;
~JsonScope() {
if (jb_) {
leave();
}
}
void leave() {
CHECK(is_active());
jb_->scope_ = save_scope_;
}
protected:
StringBuilder *sb_;
// For CHECK
JsonBuilder *jb_;
JsonScope *save_scope_;
bool is_active() const {
return jb_ && jb_->scope_ == this;
}
JsonScope &operator<<(JsonTrue x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(JsonFalse x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(JsonNull x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonBool &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonInt &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonLong &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonFloat &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonString &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonRawString &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(const JsonRaw &x) {
*sb_ << x;
return *this;
}
JsonScope &operator<<(bool x) = delete;
JsonScope &operator<<(int32 x) {
return *this << JsonInt(x);
}
JsonScope &operator<<(int64 x) {
return *this << JsonLong(x);
}
JsonScope &operator<<(double x) {
return *this << JsonFloat(x);
}
template <size_t N>
JsonScope &operator<<(const char (&x)[N]) {
return *this << JsonString(Slice(x));
}
JsonScope &operator<<(const char *x) {
return *this << JsonString(Slice(x));
}
JsonScope &operator<<(Slice x) {
return *this << JsonString(x);
}
};
class JsonValueScope final : public JsonScope {
public:
using JsonScope::JsonScope;
template <class T>
std::enable_if_t<std::is_base_of<Jsonable, typename std::decay<T>::type>::value, JsonValueScope &> operator<<(
const T &x) {
x.store(this);
return *this;
}
template <class T>
std::enable_if_t<!std::is_base_of<Jsonable, typename std::decay<T>::type>::value, JsonValueScope &> operator<<(
const T &x) {
CHECK(!was_);
was_ = true;
JsonScope::operator<<(x);
return *this;
}
JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT;
JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT;
private:
bool was_ = false;
};
class JsonArrayScope final : public JsonScope {
public:
explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) {
jb->inc_offset();
*sb_ << "[";
}
JsonArrayScope(const JsonArrayScope &) = delete;
JsonArrayScope &operator=(const JsonArrayScope &) = delete;
JsonArrayScope(JsonArrayScope &&) = default;
JsonArrayScope &operator=(JsonArrayScope &&) = delete;
~JsonArrayScope() {
if (jb_) {
leave();
}
}
void leave() {
jb_->dec_offset();
jb_->print_offset();
*sb_ << "]";
}
template <class T>
JsonArrayScope &operator<<(const T &x) {
return (*this)(x);
}
template <class T>
JsonArrayScope &operator()(const T &x) {
enter_value() << x;
return *this;
}
JsonValueScope enter_value() {
CHECK(is_active());
if (is_first_) {
*sb_ << ",";
} else {
is_first_ = true;
}
jb_->print_offset();
return jb_->enter_value();
}
private:
bool is_first_ = false;
};
class JsonObjectScope final : public JsonScope {
public:
explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) {
jb->inc_offset();
*sb_ << "{";
}
JsonObjectScope(const JsonObjectScope &) = delete;
JsonObjectScope &operator=(const JsonObjectScope &) = delete;
JsonObjectScope(JsonObjectScope &&) = default;
JsonObjectScope &operator=(JsonObjectScope &&) = delete;
~JsonObjectScope() {
if (jb_) {
leave();
}
}
void leave() {
jb_->dec_offset();
jb_->print_offset();
*sb_ << "}";
}
template <class T>
JsonObjectScope &operator()(Slice field, T &&value) {
CHECK(is_active());
if (is_first_) {
*sb_ << ",";
} else {
is_first_ = true;
}
jb_->print_offset();
jb_->enter_value() << field;
if (jb_->is_pretty()) {
*sb_ << " : ";
} else {
*sb_ << ":";
}
jb_->enter_value() << value;
return *this;
}
JsonObjectScope &operator<<(const JsonRaw &field_value) {
CHECK(is_active());
is_first_ = true;
jb_->enter_value() << field_value;
return *this;
}
private:
bool is_first_ = false;
};
inline JsonArrayScope JsonValueScope::enter_array() {
CHECK(!was_);
was_ = true;
return JsonArrayScope(jb_);
}
inline JsonObjectScope JsonValueScope::enter_object() {
CHECK(!was_);
was_ = true;
return JsonObjectScope(jb_);
}
inline JsonValueScope JsonBuilder::enter_value() {
return JsonValueScope(this);
}
inline JsonObjectScope JsonBuilder::enter_object() {
return JsonObjectScope(this);
}
inline JsonArrayScope JsonBuilder::enter_array() {
return JsonArrayScope(this);
}
class JsonValue;
enum class JsonValueType { Null, Number, Boolean, String, Array, Object };
using JsonArray = vector<JsonValue>;
class JsonObject {
const JsonValue *get_field(Slice name) const;
public:
vector<std::pair<Slice, JsonValue>> field_values_;
JsonObject() = default;
explicit JsonObject(vector<std::pair<Slice, JsonValue>> &&field_values);
JsonObject(const JsonObject &) = delete;
JsonObject &operator=(const JsonObject &) = delete;
JsonObject(JsonObject &&) = default;
JsonObject &operator=(JsonObject &&) = default;
~JsonObject() = default;
size_t field_count() const;
JsonValue extract_field(Slice name);
Result<JsonValue> extract_optional_field(Slice name, JsonValueType type);
Result<JsonValue> extract_required_field(Slice name, JsonValueType type);
bool has_field(Slice name) const;
Result<bool> get_optional_bool_field(Slice name, bool default_value = false) const;
Result<bool> get_required_bool_field(Slice name) const;
Result<int32> get_optional_int_field(Slice name, int32 default_value = 0) const;
Result<int32> get_required_int_field(Slice name) const;
Result<int64> get_optional_long_field(Slice name, int64 default_value = 0) const;
Result<int64> get_required_long_field(Slice name) const;
Result<double> get_optional_double_field(Slice name, double default_value = 0.0) const;
Result<double> get_required_double_field(Slice name) const;
Result<string> get_optional_string_field(Slice name, string default_value = string()) const;
Result<string> get_required_string_field(Slice name) const;
void foreach(const std::function<void(Slice name, const JsonValue &value)> &callback) const;
};
class JsonValue final : private Jsonable {
public:
using Type = JsonValueType;
static Slice get_type_name(Type type);
JsonValue() {
}
~JsonValue() {
destroy();
}
JsonValue(JsonValue &&other) noexcept : JsonValue() {
init(std::move(other));
}
JsonValue &operator=(JsonValue &&other) noexcept {
if (&other == this) {
return *this;
}
destroy();
init(std::move(other));
return *this;
}
JsonValue(const JsonValue &) = delete;
JsonValue &operator=(const JsonValue &) = delete;
Type type() const {
return type_;
}
MutableSlice &get_string() {
CHECK(type_ == Type::String);
return string_;
}
const MutableSlice &get_string() const {
CHECK(type_ == Type::String);
return string_;
}
bool &get_boolean() {
CHECK(type_ == Type::Boolean);
return boolean_;
}
const bool &get_boolean() const {
CHECK(type_ == Type::Boolean);
return boolean_;
}
MutableSlice &get_number() {
CHECK(type_ == Type::Number);
return number_;
}
const MutableSlice &get_number() const {
CHECK(type_ == Type::Number);
return number_;
}
JsonArray &get_array() {
CHECK(type_ == Type::Array);
return array_;
}
const JsonArray &get_array() const {
CHECK(type_ == Type::Array);
return array_;
}
JsonObject &get_object() {
CHECK(type_ == Type::Object);
return object_;
}
const JsonObject &get_object() const {
CHECK(type_ == Type::Object);
return object_;
}
static JsonValue create_boolean(bool val) {
JsonValue res;
res.init_boolean(val);
return res;
}
static JsonValue create_number(MutableSlice number) {
JsonValue res;
res.init_number(number);
return res;
}
static JsonValue create_string(MutableSlice str) {
JsonValue res;
res.init_string(str);
return res;
}
static JsonValue create_array(JsonArray v) {
JsonValue res;
res.init_array(std::move(v));
return res;
}
static JsonValue make_object(JsonObject c) {
JsonValue res;
res.init_object(std::move(c));
return res;
}
void store(JsonValueScope *scope) const {
switch (type_) {
case Type::Null:
*scope << JsonRaw("null");
break;
case Type::Boolean:
if (get_boolean()) {
*scope << JsonRaw("true");
} else {
*scope << JsonRaw("false");
}
break;
case Type::Number:
*scope << JsonRaw(get_number());
break;
case Type::String:
*scope << JsonString(get_string());
break;
case Type::Array: {
auto arr = scope->enter_array();
for (auto &val : get_array()) {
arr << val;
}
break;
}
case Type::Object: {
auto object = scope->enter_object();
for (auto &field_value : get_object().field_values_) {
object(field_value.first, field_value.second);
}
break;
}
}
};
private:
Type type_{Type::Null};
union {
MutableSlice number_;
bool boolean_;
MutableSlice string_;
JsonArray array_;
JsonObject object_;
};
void init_null() {
type_ = Type::Null;
}
void init_number(MutableSlice number) {
type_ = Type::Number;
new (&number_) MutableSlice(number);
}
void init_boolean(bool boolean) {
type_ = Type::Boolean;
boolean_ = boolean;
}
void init_string(MutableSlice slice) {
type_ = Type::String;
new (&string_) MutableSlice(slice);
}
void init_array(JsonArray array) {
type_ = Type::Array;
new (&array_) JsonArray(std::move(array));
}
void init_object(JsonObject object) {
type_ = Type::Object;
new (&object_) JsonObject(std::move(object));
}
void init(JsonValue &&other) {
switch (other.type_) {
case Type::Null:
break;
case Type::Number:
init_number(other.number_);
break;
case Type::Boolean:
init_boolean(other.boolean_);
break;
case Type::String:
init_string(other.string_);
break;
case Type::Array:
init_array(std::move(other.array_));
break;
case Type::Object:
init_object(std::move(other.object_));
break;
}
other.destroy();
}
void destroy() {
switch (type_) {
case Type::Null:
case Type::Boolean:
break;
case Type::Number:
number_.~MutableSlice();
break;
case Type::String:
string_.~MutableSlice();
break;
case Type::Array:
array_.~vector<JsonValue>();
break;
case Type::Object:
object_.~JsonObject();
break;
}
type_ = Type::Null;
}
};
inline StringBuilder &operator<<(StringBuilder &sb, JsonValue::Type type) {
switch (type) {
case JsonValue::Type::Null:
return sb << "Null";
case JsonValue::Type::Number:
return sb << "Number";
case JsonValue::Type::Boolean:
return sb << "Boolean";
case JsonValue::Type::String:
return sb << "String";
case JsonValue::Type::Array:
return sb << "Array";
case JsonValue::Type::Object:
return sb << "Object";
default:
UNREACHABLE();
return sb;
}
}
class VirtuallyJsonable : private Jsonable {
public:
virtual void store(JsonValueScope *scope) const = 0;
VirtuallyJsonable() = default;
VirtuallyJsonable(const VirtuallyJsonable &) = delete;
VirtuallyJsonable &operator=(const VirtuallyJsonable &) = delete;
VirtuallyJsonable(VirtuallyJsonable &&) = default;
VirtuallyJsonable &operator=(VirtuallyJsonable &&) = default;
virtual ~VirtuallyJsonable() = default;
};
class VirtuallyJsonableInt final : public VirtuallyJsonable {
public:
explicit VirtuallyJsonableInt(int32 value) : value_(value) {
}
void store(JsonValueScope *scope) const final {
*scope << JsonInt(value_);
}
private:
int32 value_;
};
class VirtuallyJsonableLong final : public VirtuallyJsonable {
public:
explicit VirtuallyJsonableLong(int64 value) : value_(value) {
}
void store(JsonValueScope *scope) const final {
*scope << JsonLong(value_);
}
private:
int64 value_;
};
class VirtuallyJsonableString final : public VirtuallyJsonable {
public:
explicit VirtuallyJsonableString(Slice value) : value_(value) {
}
void store(JsonValueScope *scope) const final {
*scope << JsonString(value_);
}
private:
Slice value_;
};
Result<MutableSlice> json_string_decode(Parser &parser) TD_WARN_UNUSED_RESULT;
Status json_string_skip(Parser &parser) TD_WARN_UNUSED_RESULT;
Result<JsonValue> do_json_decode(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT;
Status do_json_skip(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT;
inline Result<JsonValue> json_decode(MutableSlice json) {
Parser parser(json);
const int32 DEFAULT_MAX_DEPTH = 100;
auto result = do_json_decode(parser, DEFAULT_MAX_DEPTH);
if (result.is_ok()) {
parser.skip_whitespaces();
if (!parser.empty()) {
return Status::Error("Expected string end");
}
}
return result;
}
template <class StrT, class ValT>
StrT json_encode(const ValT &val, bool pretty = false) {
auto buf_len = 1 << 18;
auto buf = StackAllocator::alloc(buf_len);
JsonBuilder jb(StringBuilder(buf.as_slice(), true), pretty ? 0 : -1);
jb.enter_value() << val;
if (pretty) {
jb.string_builder() << "\n";
}
LOG_IF(ERROR, jb.string_builder().is_error()) << "JSON buffer overflow";
auto slice = jb.string_builder().as_cslice();
return StrT(slice.begin(), slice.size());
}
template <class T>
class ToJsonImpl final : private Jsonable {
public:
explicit ToJsonImpl(const T &value) : value_(value) {
}
void store(JsonValueScope *scope) const {
to_json(*scope, value_);
}
private:
const T &value_;
};
template <class T>
auto ToJson(const T &value) {
return ToJsonImpl<T>(value);
}
template <class T>
void to_json(JsonValueScope &jv, const T &value) {
jv << value;
}
template <class F>
class JsonObjectImpl : private Jsonable {
public:
explicit JsonObjectImpl(F &&f) : f_(std::forward<F>(f)) {
}
void store(JsonValueScope *scope) const {
auto object = scope->enter_object();
f_(object);
}
private:
F f_;
};
template <class F>
auto json_object(F &&f) {
return JsonObjectImpl<F>(std::forward<F>(f));
}
template <class F>
class JsonArrayImpl : private Jsonable {
public:
explicit JsonArrayImpl(F &&f) : f_(std::forward<F>(f)) {
}
void store(JsonValueScope *scope) const {
auto array = scope->enter_array();
f_(array);
}
private:
F f_;
};
template <class F>
auto json_array(F &&f) {
return JsonArrayImpl<F>(std::forward<F>(f));
}
template <class A, class F>
auto json_array(const A &a, F &&f) {
return json_array([&a, &f](auto &arr) {
for (auto &x : a) {
arr(f(x));
}
});
}
} // namespace td

129
td/tdutils/td/utils/List.h Normal file
View File

@@ -0,0 +1,129 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
namespace td {
struct ListNode {
ListNode *next;
ListNode *prev;
ListNode() {
clear();
}
~ListNode() {
remove();
}
ListNode(const ListNode &) = delete;
ListNode &operator=(const ListNode &) = delete;
ListNode(ListNode &&other) noexcept {
if (other.empty()) {
clear();
} else {
init_from(std::move(other));
}
}
ListNode &operator=(ListNode &&other) noexcept {
if (this == &other) {
return *this;
}
this->remove();
if (!other.empty()) {
init_from(std::move(other));
}
return *this;
}
void connect(ListNode *to) {
CHECK(to != nullptr);
next = to;
to->prev = this;
}
void remove() {
prev->connect(next);
clear();
}
void put(ListNode *other) {
DCHECK(other->empty());
put_unsafe(other);
}
void put_back(ListNode *other) {
DCHECK(other->empty());
prev->connect(other);
other->connect(this);
}
ListNode *get() {
ListNode *result = prev;
if (result == this) {
return nullptr;
}
result->prev->connect(this);
result->clear();
// this->connect(result->next);
return result;
}
bool empty() const {
return next == this;
}
ListNode *begin() {
return next;
}
ListNode *end() {
return this;
}
const ListNode *begin() const {
return next;
}
const ListNode *end() const {
return this;
}
ListNode *get_next() {
return next;
}
ListNode *get_prev() {
return prev;
}
const ListNode *get_next() const {
return next;
}
const ListNode *get_prev() const {
return prev;
}
protected:
void clear() {
next = this;
prev = this;
}
void init_from(ListNode &&other) {
ListNode *head = other.prev;
other.remove();
head->put_unsafe(this);
}
void put_unsafe(ListNode *other) {
other->connect(next);
this->connect(other);
}
};
} // namespace td

166
td/tdutils/td/utils/MapNode.h Normal file
View File

@@ -0,0 +1,166 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/HashTableUtils.h"
#include <new>
#include <type_traits>
#include <utility>
namespace td {
template <class KeyT, class ValueT, class EqT, class Enable = void>
struct MapNode {
using first_type = KeyT;
using second_type = ValueT;
using public_key_type = KeyT;
using public_type = MapNode;
KeyT first{};
union {
ValueT second;
};
const KeyT &key() const {
return first;
}
MapNode &get_public() {
return *this;
}
const MapNode &get_public() const {
return *this;
}
MapNode() {
}
MapNode(KeyT key, ValueT value) : first(std::move(key)) {
new (&second) ValueT(std::move(value));
DCHECK(!empty());
}
MapNode(const MapNode &) = delete;
MapNode &operator=(const MapNode &) = delete;
MapNode(MapNode &&other) noexcept {
*this = std::move(other);
}
void operator=(MapNode &&other) noexcept {
DCHECK(empty());
DCHECK(!other.empty());
first = std::move(other.first);
other.first = KeyT();
new (&second) ValueT(std::move(other.second));
other.second.~ValueT();
}
~MapNode() {
if (!empty()) {
second.~ValueT();
}
}
void copy_from(const MapNode &other) {
DCHECK(empty());
DCHECK(!other.empty());
first = other.first;
new (&second) ValueT(other.second);
}
bool empty() const {
return is_hash_table_key_empty<EqT>(first);
}
void clear() {
DCHECK(!empty());
first = KeyT();
second.~ValueT();
DCHECK(empty());
}
template <class... ArgsT>
void emplace(KeyT key, ArgsT &&...args) {
DCHECK(empty());
first = std::move(key);
new (&second) ValueT(std::forward<ArgsT>(args)...);
DCHECK(!empty());
}
};
template <class KeyT, class ValueT, class EqT>
struct MapNode<KeyT, ValueT, EqT, typename std::enable_if_t<(sizeof(KeyT) + sizeof(ValueT) > 28 * sizeof(void *))>> {
struct Impl {
using first_type = KeyT;
using second_type = ValueT;
KeyT first{};
union {
ValueT second;
};
template <class InputKeyT, class... ArgsT>
Impl(InputKeyT &&key, ArgsT &&...args) : first(std::forward<InputKeyT>(key)) {
new (&second) ValueT(std::forward<ArgsT>(args)...);
DCHECK(!is_hash_table_key_empty<EqT>(first));
}
Impl(const Impl &) = delete;
Impl &operator=(const Impl &) = delete;
Impl(Impl &&) = delete;
Impl &operator=(Impl &&) = delete;
~Impl() {
second.~ValueT();
}
};
using first_type = KeyT;
using second_type = ValueT;
using public_key_type = KeyT;
using public_type = Impl;
unique_ptr<Impl> impl_;
const KeyT &key() const {
DCHECK(!empty());
return impl_->first;
}
Impl &get_public() {
return *impl_;
}
const Impl &get_public() const {
return *impl_;
}
MapNode() {
}
MapNode(KeyT key, ValueT value) : impl_(td::make_unique<Impl>(std::move(key), std::move(value))) {
}
void copy_from(const MapNode &other) {
DCHECK(empty());
DCHECK(!other.empty());
impl_ = td::make_unique<Impl>(other.impl_->first, other.impl_->second);
}
bool empty() const {
return impl_ == nullptr;
}
void clear() {
DCHECK(!empty());
impl_ = nullptr;
}
template <class... ArgsT>
void emplace(KeyT key, ArgsT &&...args) {
DCHECK(empty());
impl_ = td::make_unique<Impl>(std::move(key), std::forward<ArgsT>(args)...);
}
};
} // namespace td

79
td/tdutils/td/utils/MemoryLog.h Normal file
View File

@@ -0,0 +1,79 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/Slice.h"
#include <atomic>
#include <cstdio>
#include <cstring>
namespace td {
template <int buffer_size = 32 * (1 << 10)>
class MemoryLog final : public LogInterface {
static constexpr size_t MAX_OUTPUT_SIZE = buffer_size / 16 < (8 << 10) ? buffer_size / 16 : (8 << 10);
static_assert((buffer_size & (buffer_size - 1)) == 0, "Buffer size must be power of 2");
static_assert(buffer_size >= (8 << 10), "Too small buffer size");
public:
MemoryLog() {
std::memset(buffer_, ' ', sizeof(buffer_));
}
Slice get_buffer() const {
return Slice(buffer_, sizeof(buffer_));
}
size_t get_pos() const {
return pos_ & (buffer_size - 1);
}
private:
void do_append(int log_level, CSlice new_slice) final {
Slice slice = new_slice;
slice.truncate(MAX_OUTPUT_SIZE);
while (!slice.empty() && slice.back() == '\n') {
slice.remove_suffix(1);
}
size_t slice_size = slice.size();
CHECK(slice_size * 3 < buffer_size);
size_t pad_size = ((slice_size + 15) & ~15) - slice_size;
constexpr size_t MAGIC_SIZE = 16;
auto total_size = static_cast<uint32>(slice_size + pad_size + MAGIC_SIZE);
auto real_pos = pos_.fetch_add(total_size, std::memory_order_relaxed);
CHECK((total_size & 15) == 0);
uint32 start_pos = real_pos & (buffer_size - 1);
uint32 end_pos = start_pos + total_size;
if (likely(end_pos <= buffer_size)) {
std::memcpy(&buffer_[start_pos + MAGIC_SIZE], slice.data(), slice_size);
std::memcpy(&buffer_[start_pos + MAGIC_SIZE + slice_size], " ", pad_size);
} else {
size_t first = buffer_size - start_pos - MAGIC_SIZE;
size_t second = slice_size - first;
std::memcpy(&buffer_[start_pos + MAGIC_SIZE], slice.data(), first);
std::memcpy(&buffer_[0], slice.data() + first, second);
std::memcpy(&buffer_[second], " ", pad_size);
}
CHECK((start_pos & 15) == 0);
CHECK(start_pos <= buffer_size - MAGIC_SIZE);
buffer_[start_pos] = '\n';
size_t printed = std::snprintf(&buffer_[start_pos + 1], MAGIC_SIZE - 1, "LOG:%08x: ", real_pos);
CHECK(printed == MAGIC_SIZE - 2);
buffer_[start_pos + MAGIC_SIZE - 1] = ' ';
}
char buffer_[buffer_size];
std::atomic<uint32> pos_{0};
};
} // namespace td

44
td/tdutils/td/utils/MimeType.cpp Normal file
View File

@@ -0,0 +1,44 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/MimeType.h"
#include "td/utils/logging.h"
const char *extension_to_mime_type(const char *extension, size_t extension_len); // auto-generated
const char *mime_type_to_extension(const char *mime_type, size_t mime_type_len); // auto-generated
namespace td {
string MimeType::to_extension(Slice mime_type, Slice default_value) {
if (mime_type.empty()) {
return default_value.str();
}
const char *result = ::mime_type_to_extension(mime_type.data(), mime_type.size());
if (result != nullptr) {
return result;
}
LOG(INFO) << "Unknown file MIME type " << mime_type;
return default_value.str();
}
string MimeType::from_extension(Slice extension, Slice default_value) {
if (extension.empty()) {
return default_value.str();
}
const char *result = ::extension_to_mime_type(extension.data(), extension.size());
if (result != nullptr) {
return result;
}
LOG(INFO) << "Unknown file extension " << extension;
return default_value.str();
}
} // namespace td

20
td/tdutils/td/utils/MimeType.h Normal file
View File

@@ -0,0 +1,20 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
namespace td {
class MimeType {
public:
static string to_extension(Slice mime_type, Slice default_value = Slice());
static string from_extension(Slice extension, Slice default_value = Slice());
};
} // namespace td

42
td/tdutils/td/utils/MovableValue.h Normal file
View File

@@ -0,0 +1,42 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
namespace td {
template <class T, T empty_val = T()>
class MovableValue {
public:
MovableValue() = default;
MovableValue(T val) : val_(val) {
}
MovableValue(MovableValue &&other) noexcept : val_(other.val_) {
other.clear();
}
MovableValue &operator=(MovableValue &&other) noexcept {
if (this != &other) {
val_ = other.val_;
other.clear();
}
return *this;
}
MovableValue(const MovableValue &) = default;
MovableValue &operator=(const MovableValue &) = default;
~MovableValue() = default;
void clear() {
val_ = empty_val;
}
const T &get() const {
return val_;
}
private:
T val_ = empty_val;
};
} // namespace td

15
td/tdutils/td/utils/MpmcQueue.cpp Normal file
View File

@@ -0,0 +1,15 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/MpmcQueue.h"
namespace td {
namespace detail {
MpmcStat stat_;
} // namespace detail
} // namespace td

460
td/tdutils/td/utils/MpmcQueue.h Normal file
View File

@@ -0,0 +1,460 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
// MPMC queue
// Simple semaphore protected implementation
// To close queue, one should send as much sentinel elements as there are readers.
// Once there are no readers and writers, one may easily destroy queue
#include "td/utils/format.h"
#include "td/utils/HazardPointers.h"
#include "td/utils/logging.h"
#include "td/utils/port/sleep.h"
#include "td/utils/ScopeGuard.h"
#include <array>
#include <atomic>
namespace td {
namespace detail {
struct MpmcStat {
void alloc_ok(size_t thread_id) {
s(thread_id).alloc_ok_cnt++;
}
void alloc_error(size_t thread_id) {
s(thread_id).alloc_error_cnt++;
}
void push_loop_error(size_t thread_id) {
s(thread_id).push_loop_error_cnt++;
}
void push_loop_ok(size_t thread_id) {
s(thread_id).push_loop_ok_cnt++;
}
void dump() {
int alloc_ok_cnt = 0;
int alloc_error_cnt = 0;
int push_loop_error_cnt = 0;
int push_loop_ok_cnt = 0;
for (auto &d : arr) {
alloc_ok_cnt += d.alloc_ok_cnt;
alloc_error_cnt += d.alloc_error_cnt;
push_loop_error_cnt += d.push_loop_error_cnt;
push_loop_ok_cnt += d.push_loop_ok_cnt;
}
LOG(ERROR) << tag("alloc_ok_cnt", alloc_ok_cnt) << tag("alloc_error_cnt", alloc_error_cnt)
<< tag("push_loop_error_cnt", push_loop_error_cnt) << tag("push_loop_ok_cnt", push_loop_ok_cnt);
}
private:
struct ThreadStat {
int alloc_ok_cnt{0};
int alloc_error_cnt{0};
int push_loop_ok_cnt{0};
int push_loop_error_cnt{0};
char pad[TD_CONCURRENCY_PAD - sizeof(int) * 4];
};
std::array<ThreadStat, 1024> arr;
ThreadStat &s(size_t thread_id) {
return arr[thread_id];
}
};
extern MpmcStat stat_;
} // namespace detail
template <class T>
class OneValue {
public:
bool set_value(T &value) {
value_ = std::move(value);
int state = Empty;
if (state_.compare_exchange_strong(state, Value, std::memory_order_acq_rel)) {
return true;
}
value = std::move(value_);
return false;
}
bool get_value(T &value) {
auto old_state = state_.exchange(Taken, std::memory_order_acq_rel);
if (old_state == Value) {
value = std::move(value_);
return true;
}
return false;
}
void reset() {
state_ = Empty;
value_ = T();
}
private:
enum Type : int { Empty = 0, Taken, Value };
std::atomic<int> state_{Empty};
T value_{};
};
template <class T>
class OneValue<T *> {
public:
bool set_value(T *value) {
T *was = Empty();
return state_.compare_exchange_strong(was, value, std::memory_order_acq_rel);
}
bool get_value(T *&value) {
value = state_.exchange(Taken(), std::memory_order_acq_rel);
return value != Empty();
}
void reset() {
state_ = Empty();
}
OneValue() {
}
private:
std::atomic<T *> state_{Empty()};
static T *Empty() {
static int64 xxx;
return reinterpret_cast<T *>(&xxx);
}
static T *Taken() {
static int64 xxx;
return reinterpret_cast<T *>(&xxx);
}
};
template <class T>
class MpmcQueueBlock {
public:
explicit MpmcQueueBlock(size_t size) : nodes_(size) {
}
enum class PopStatus { Ok, Empty, Closed };
//blocking pop
//returns Ok or Closed
PopStatus pop(T &value) {
while (true) {
auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed);
if (read_pos >= nodes_.size()) {
return PopStatus::Closed;
}
//TODO blocking get_value
if (nodes_[static_cast<size_t>(read_pos)].one_value.get_value(value)) {
return PopStatus::Ok;
}
}
}
//nonblocking pop
//returns Ok, Empty or Closed
PopStatus try_pop(T &value) {
while (true) {
// this check slows 1:1 case but prevents writer starvation in 1:N case
if (write_pos_.load(std::memory_order_relaxed) <= read_pos_.load(std::memory_order_relaxed) &&
read_pos_.load(std::memory_order_relaxed) < nodes_.size()) {
return PopStatus::Empty;
}
auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed);
if (read_pos >= nodes_.size()) {
return PopStatus::Closed;
}
if (nodes_[static_cast<size_t>(read_pos)].one_value.get_value(value)) {
return PopStatus::Ok;
}
auto write_pos = write_pos_.load(std::memory_order_relaxed);
if (write_pos <= read_pos + 1) {
return PopStatus::Empty;
}
}
}
enum class PushStatus { Ok, Closed };
PushStatus push(T &value) {
while (true) {
auto write_pos = write_pos_.fetch_add(1, std::memory_order_relaxed);
if (write_pos >= nodes_.size()) {
return PushStatus::Closed;
}
if (nodes_[static_cast<size_t>(write_pos)].one_value.set_value(value)) {
//stat_.push_loop_ok(0);
return PushStatus::Ok;
}
//stat_.push_loop_error(0);
}
}
private:
struct Node {
OneValue<T> one_value;
};
std::atomic<uint64> write_pos_{0};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::atomic<uint64> read_pos_{0};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::vector<Node> nodes_;
char pad3[TD_CONCURRENCY_PAD - sizeof(std::vector<Node>)];
};
template <class T>
class MpmcQueueOld {
public:
explicit MpmcQueueOld(size_t threads_n) : MpmcQueueOld(1024, threads_n) {
}
static std::string get_description() {
return "Mpmc queue (fetch and add array queue)";
}
MpmcQueueOld(size_t block_size, size_t threads_n) : block_size_{block_size}, hazard_pointers_{threads_n} {
auto node = make_unique<Node>(block_size_);
write_pos_ = node.get();
read_pos_ = node.get();
node.release();
}
MpmcQueueOld(const MpmcQueueOld &) = delete;
MpmcQueueOld &operator=(const MpmcQueueOld &) = delete;
MpmcQueueOld(MpmcQueueOld &&) = delete;
MpmcQueueOld &operator=(MpmcQueueOld &&) = delete;
~MpmcQueueOld() {
auto *ptr = read_pos_.load(std::memory_order_relaxed);
while (ptr) {
auto *to_delete = ptr;
ptr = ptr->next_.load(std::memory_order_relaxed);
delete to_delete;
}
//stat_.dump();
//stat_ = detail::MpmcStat();
}
size_t hazard_pointers_to_delele_size_unsafe() const {
return hazard_pointers_.to_delete_size_unsafe();
}
void gc(size_t thread_id) {
hazard_pointers_.retire(thread_id);
}
using PushStatus = typename MpmcQueueBlock<T>::PushStatus;
using PopStatus = typename MpmcQueueBlock<T>::PopStatus;
void push(T value, size_t thread_id) {
typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0);
while (true) {
auto node = hazard_ptr_holder.protect(write_pos_);
auto status = node->block.push(value);
switch (status) {
case PushStatus::Ok:
return;
case PushStatus::Closed: {
auto next = node->next_.load(std::memory_order_acquire);
if (next == nullptr) {
auto new_node = new Node(block_size_);
new_node->block.push(value);
if (node->next_.compare_exchange_strong(next, new_node, std::memory_order_acq_rel)) {
//stat_.alloc_ok(thread_id);
write_pos_.compare_exchange_strong(node, new_node, std::memory_order_acq_rel);
return;
} else {
//stat_.alloc_error(thread_id);
new_node->block.pop(value);
//CHECK(status == PopStatus::Ok);
delete new_node;
}
}
//CHECK(next != nullptr);
write_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel);
break;
}
}
}
}
bool try_pop(T &value, size_t thread_id) {
typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0);
while (true) {
auto node = hazard_ptr_holder.protect(read_pos_);
auto status = node->block.try_pop(value);
switch (status) {
case PopStatus::Ok:
return true;
case PopStatus::Empty:
return false;
case PopStatus::Closed: {
auto next = node->next_.load(std::memory_order_acquire);
if (!next) {
return false;
}
if (read_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel)) {
hazard_ptr_holder.clear();
hazard_pointers_.retire(thread_id, node);
}
break;
}
}
}
}
T pop(size_t thread_id) {
T value;
while (true) {
if (try_pop(value, thread_id)) {
return value;
}
usleep_for(1);
}
}
private:
struct Node {
explicit Node(size_t block_size) : block{block_size} {
}
std::atomic<Node *> next_{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
MpmcQueueBlock<T> block;
// MpmcQueueBlock is already padded
};
std::atomic<Node *> write_pos_{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
std::atomic<Node *> read_pos_{nullptr};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
size_t block_size_;
HazardPointers<Node, 1> hazard_pointers_;
// HazardPointers class is already padded
};
template <class T>
class MpmcQueue {
public:
explicit MpmcQueue(size_t threads_n) : MpmcQueue(1024, threads_n) {
}
static std::string get_description() {
return "NEW Mpmc queue (fetch and add array queue)";
}
MpmcQueue(size_t block_size, size_t threads_n) : hazard_pointers_{threads_n} {
auto node = make_unique<Node>();
write_pos_ = node.get();
read_pos_ = node.get();
node.release();
}
MpmcQueue(const MpmcQueue &) = delete;
MpmcQueue &operator=(const MpmcQueue &) = delete;
MpmcQueue(MpmcQueue &&) = delete;
MpmcQueue &operator=(MpmcQueue &&) = delete;
~MpmcQueue() {
auto *ptr = read_pos_.load(std::memory_order_relaxed);
while (ptr) {
auto *to_delete = ptr;
ptr = ptr->next.load(std::memory_order_relaxed);
delete to_delete;
}
}
size_t hazard_pointers_to_delele_size_unsafe() const {
return hazard_pointers_.to_delete_size_unsafe();
}
void gc(size_t thread_id) {
hazard_pointers_.retire(thread_id);
}
void push(T value, size_t thread_id) {
SCOPE_EXIT {
hazard_pointers_.clear(thread_id, 0);
};
while (true) {
auto node = hazard_pointers_.protect(thread_id, 0, write_pos_);
auto &block = node->block;
auto pos = block.write_pos++;
if (pos >= block.data.size()) {
auto next = node->next.load();
if (next == nullptr) {
auto new_node = new Node{};
new_node->block.write_pos++;
new_node->block.data[0].set_value(value);
Node *null = nullptr;
if (node->next.compare_exchange_strong(null, new_node)) {
write_pos_.compare_exchange_strong(node, new_node);
return;
} else {
new_node->block.data[0].get_value(value);
delete new_node;
}
} else {
write_pos_.compare_exchange_strong(node, next);
}
} else {
if (block.data[static_cast<size_t>(pos)].set_value(value)) {
return;
}
}
}
}
bool try_pop(T &value, size_t thread_id) {
SCOPE_EXIT {
hazard_pointers_.clear(thread_id, 0);
};
while (true) {
auto node = hazard_pointers_.protect(thread_id, 0, read_pos_);
auto &block = node->block;
if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) {
return false;
}
auto pos = block.read_pos++;
if (pos >= block.data.size()) {
auto next = node->next.load();
if (!next) {
return false;
}
if (read_pos_.compare_exchange_strong(node, next)) {
hazard_pointers_.clear(thread_id, 0);
hazard_pointers_.retire(thread_id, node);
}
} else {
if (block.data[static_cast<size_t>(pos)].get_value(value)) {
return true;
}
}
}
}
T pop(size_t thread_id) {
T value;
while (true) {
if (try_pop(value, thread_id)) {
return value;
}
usleep_for(1);
}
}
private:
struct Block {
std::atomic<uint64> write_pos{0};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::atomic<uint64> read_pos{0};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)];
std::array<OneValue<T>, 1024> data;
char pad3[TD_CONCURRENCY_PAD];
};
struct Node {
Node() = default;
Block block;
std::atomic<Node *> next{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
};
std::atomic<Node *> write_pos_{nullptr};
char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
std::atomic<Node *> read_pos_{nullptr};
char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)];
HazardPointers<Node, 1> hazard_pointers_;
// HazardPointers class is already padded
};
} // namespace td

336
td/tdutils/td/utils/MpmcWaiter.h Normal file
View File

@@ -0,0 +1,336 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/sleep.h"
#include <algorithm>
#include <atomic>
#include <condition_variable>
#include <mutex>
namespace td {
class MpmcEagerWaiter {
public:
struct Slot {
private:
friend class MpmcEagerWaiter;
int yields;
uint32 worker_id;
};
static void init_slot(Slot &slot, uint32 worker_id) {
slot.yields = 0;
slot.worker_id = worker_id;
}
void wait(Slot &slot) {
if (slot.yields < RoundsTillSleepy) {
yield();
slot.yields++;
} else if (slot.yields == RoundsTillSleepy) {
auto state = state_.load(std::memory_order_relaxed);
if (!State::has_worker(state)) {
auto new_state = State::with_worker(state, slot.worker_id);
if (state_.compare_exchange_strong(state, new_state, std::memory_order_acq_rel)) {
yield();
slot.yields++;
return;
}
if (state == State::awake()) {
slot.yields = 0;
return;
}
}
yield();
slot.yields = 0;
} else if (slot.yields < RoundsTillAsleep) {
auto state = state_.load(std::memory_order_acquire);
if (State::still_sleepy(state, slot.worker_id)) {
yield();
slot.yields++;
return;
}
slot.yields = 0;
} else {
auto state = state_.load(std::memory_order_acquire);
if (State::still_sleepy(state, slot.worker_id)) {
std::unique_lock<std::mutex> lock(mutex_);
if (state_.compare_exchange_strong(state, State::asleep(), std::memory_order_acq_rel)) {
condition_variable_.wait(lock);
}
}
slot.yields = 0;
}
}
void stop_wait(Slot &slot) {
if (slot.yields > RoundsTillSleepy) {
notify_cold();
}
slot.yields = 0;
}
void close() {
}
void notify() {
std::atomic_thread_fence(std::memory_order_seq_cst);
if (state_.load(std::memory_order_acquire) == State::awake()) {
return;
}
notify_cold();
}
private:
struct State {
static constexpr uint32 awake() {
return 0;
}
static constexpr uint32 asleep() {
return 1;
}
static bool is_asleep(uint32 state) {
return (state & 1) != 0;
}
static bool has_worker(uint32 state) {
return (state >> 1) != 0;
}
static int32 with_worker(uint32 state, uint32 worker) {
return state | ((worker + 1) << 1);
}
static bool still_sleepy(uint32 state, uint32 worker) {
return (state >> 1) == (worker + 1);
}
};
enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 };
// enum { RoundsTillSleepy = 1, RoundsTillAsleep = 2 };
std::atomic<uint32> state_{State::awake()};
std::mutex mutex_;
std::condition_variable condition_variable_;
void notify_cold() {
auto old_state = state_.exchange(State::awake(), std::memory_order_release);
if (State::is_asleep(old_state)) {
std::lock_guard<std::mutex> guard(mutex_);
condition_variable_.notify_all();
}
}
static void yield() {
// whatever, this is better than sched_yield
usleep_for(1);
}
};
class MpmcSleepyWaiter {
public:
struct Slot {
private:
friend class MpmcSleepyWaiter;
enum State { Search, Work, Sleep } state_{Work};
void park() {
std::unique_lock<std::mutex> guard(mutex_);
condition_variable_.wait(guard, [&] { return unpark_flag_; });
unpark_flag_ = false;
}
bool cancel_park() {
auto res = unpark_flag_;
unpark_flag_ = false;
return res;
}
void unpark() {
//TODO: try to unlock guard before notify_all
std::unique_lock<std::mutex> guard(mutex_);
unpark_flag_ = true;
condition_variable_.notify_all();
}
std::mutex mutex_;
std::condition_variable condition_variable_;
bool unpark_flag_{false}; // TODO: move out of lock
int yield_cnt{0};
int32 worker_id{0};
public:
char padding[TD_CONCURRENCY_PAD];
};
// There are a lot of workers
// Each has a slot
//
// States of a worker:
// - searching for work | Search
// - processing work | Work
// - sleeping | Sleep
//
// When somebody adds a work it calls notify
//
// notify
// if there are workers in search phase do nothing.
// if all workers are awake do nothing
// otherwise wake some random worker
//
// Initially all workers are in Search mode.
//
// When worker found nothing it may try to call wait.
// This may put it in a Sleep for some time.
// After wait returns worker will be in Search state again.
//
// If a worker found a work and ready to process it, then it may call stop_wait.
// This will cause transition from Search to Work state.
//
// Main invariant:
// After notify is called there should be at least on worker in Search or Work state.
// If possible - in Search state
//
static void init_slot(Slot &slot, int32 worker_id) {
slot.state_ = Slot::State::Work;
slot.unpark_flag_ = false;
slot.worker_id = worker_id;
VLOG(waiter) << "Init slot " << worker_id;
}
static constexpr int VERBOSITY_NAME(waiter) = VERBOSITY_NAME(DEBUG) + 10;
void wait(Slot &slot) {
if (slot.state_ == Slot::State::Work) {
VLOG(waiter) << "Work -> Search";
state_++;
slot.state_ = Slot::State::Search;
slot.yield_cnt = 0;
return;
}
if (slot.state_ == Slot::Search) {
if (slot.yield_cnt++ < 10 && false) {
// TODO some sleep backoff is possible
return;
}
slot.state_ = Slot::State::Sleep;
std::unique_lock<std::mutex> guard(sleepers_mutex_);
auto state_view = StateView(state_.fetch_add((1 << PARKING_SHIFT) - 1));
CHECK(state_view.searching_count != 0);
bool should_search = state_view.searching_count == 1;
if (closed_) {
return;
}
sleepers_.push_back(&slot);
LOG_CHECK(slot.unpark_flag_ == false) << slot.worker_id;
VLOG(waiter) << "Add to sleepers " << slot.worker_id;
//guard.unlock();
if (should_search) {
VLOG(waiter) << "Search -> Search once, then Sleep ";
return;
}
VLOG(waiter) << "Search -> Sleep " << state_view.searching_count << " " << state_view.parked_count;
}
CHECK(slot.state_ == Slot::State::Sleep);
VLOG(waiter) << "Park " << slot.worker_id;
slot.park();
VLOG(waiter) << "Resume " << slot.worker_id;
slot.state_ = Slot::State::Search;
slot.yield_cnt = 0;
}
void stop_wait(Slot &slot) {
if (slot.state_ == Slot::State::Work) {
return;
}
if (slot.state_ == Slot::State::Sleep) {
VLOG(waiter) << "Search once, then Sleep -> Work/Search " << slot.worker_id;
slot.state_ = Slot::State::Work;
std::unique_lock<std::mutex> guard(sleepers_mutex_);
auto it = std::find(sleepers_.begin(), sleepers_.end(), &slot);
if (it != sleepers_.end()) {
sleepers_.erase(it);
VLOG(waiter) << "Remove from sleepers " << slot.worker_id;
state_.fetch_sub((1 << PARKING_SHIFT) - 1);
guard.unlock();
} else {
guard.unlock();
VLOG(waiter) << "Not in sleepers " << slot.worker_id;
CHECK(slot.cancel_park());
}
}
VLOG(waiter) << "Search once, then Sleep -> Work " << slot.worker_id;
slot.state_ = Slot::State::Search;
auto state_view = StateView(state_.fetch_sub(1));
CHECK(state_view.searching_count != 0);
CHECK(state_view.searching_count < 1000);
bool should_notify = state_view.searching_count == 1;
if (should_notify) {
VLOG(waiter) << "Notify others";
notify();
}
VLOG(waiter) << "Search -> Work ";
slot.state_ = Slot::State::Work;
}
void notify() {
auto view = StateView(state_.load());
//LOG(ERROR) << view.parked_count;
if (view.searching_count > 0 || view.parked_count == 0) {
VLOG(waiter) << "Ingore notify: " << view.searching_count << ' ' << view.parked_count;
return;
}
VLOG(waiter) << "Notify: " << view.searching_count << ' ' << view.parked_count;
std::unique_lock<std::mutex> guard(sleepers_mutex_);
view = StateView(state_.load());
if (view.searching_count > 0) {
VLOG(waiter) << "Skip notify: search is active";
return;
}
CHECK(view.parked_count == static_cast<int>(sleepers_.size()));
if (sleepers_.empty()) {
VLOG(waiter) << "Skip notify: no sleepers";
return;
}
auto sleeper = sleepers_.back();
sleepers_.pop_back();
state_.fetch_sub((1 << PARKING_SHIFT) - 1);
VLOG(waiter) << "Unpark " << sleeper->worker_id;
sleeper->unpark();
}
void close() {
StateView state(state_.load());
LOG_CHECK(state.parked_count == 0) << state.parked_count;
LOG_CHECK(state.searching_count == 0) << state.searching_count;
}
private:
static constexpr int32 PARKING_SHIFT = 16;
struct StateView {
int32 parked_count;
int32 searching_count;
explicit StateView(int32 x) {
parked_count = x >> PARKING_SHIFT;
searching_count = x & ((1 << PARKING_SHIFT) - 1);
}
};
std::atomic<int32> state_{0};
std::mutex sleepers_mutex_;
vector<Slot *> sleepers_;
bool closed_ = false;
};
using MpmcWaiter = MpmcSleepyWaiter;
} // namespace td

173
td/tdutils/td/utils/MpscLinkQueue.h Normal file
View File

@@ -0,0 +1,173 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <atomic>
namespace td {
//NB: holder of the queue holds all responsibility of freeing its nodes
class MpscLinkQueueImpl {
public:
class Node;
class Reader;
void push(Node *node) {
node->next_ = head_.load(std::memory_order_relaxed);
while (!head_.compare_exchange_strong(node->next_, node, std::memory_order_release, std::memory_order_relaxed)) {
}
}
void push_unsafe(Node *node) {
node->next_ = head_.load(std::memory_order_relaxed);
head_.store(node, std::memory_order_relaxed);
}
void pop_all(Reader &reader) {
return reader.add(head_.exchange(nullptr, std::memory_order_acquire));
}
void pop_all_unsafe(Reader &reader) {
return reader.add(head_.exchange(nullptr, std::memory_order_relaxed));
}
class Node {
friend class MpscLinkQueueImpl;
Node *next_{nullptr};
};
class Reader {
public:
Node *read() {
auto old_head = head_;
if (head_) {
head_ = head_->next_;
}
return old_head;
}
void delay(Node *node) {
node->next_ = head_;
if (!head_) {
tail_ = node;
}
head_ = node;
}
size_t calc_size() const {
size_t res = 0;
for (auto it = head_; it != nullptr; it = it->next_, res++) {
}
return res;
}
private:
friend class MpscLinkQueueImpl;
void add(Node *node) {
if (node == nullptr) {
return;
}
// Reverse list
Node *tail = node;
Node *head = nullptr;
while (node) {
auto next = node->next_;
node->next_ = head;
head = node;
node = next;
}
if (head_ == nullptr) {
head_ = head;
} else {
tail_->next_ = head;
}
tail_ = tail;
}
Node *head_{nullptr};
Node *tail_{nullptr};
};
private:
std::atomic<Node *> head_{nullptr};
};
// Uses MpscLinkQueueImpl.
// Node should have to_mpsc_link_queue_node and from_mpsc_link_queue_node functions
template <class Node>
class MpscLinkQueue {
public:
void push(Node node) {
impl_.push(node.to_mpsc_link_queue_node());
}
void push_unsafe(Node node) {
impl_.push_unsafe(node.to_mpsc_link_queue_node());
}
class Reader {
public:
~Reader() {
CHECK(!read());
}
Node read() {
auto node = impl_.read();
if (!node) {
return {};
}
return Node::from_mpsc_link_queue_node(node);
}
void delay(Node node) {
impl_.delay(node.to_mpsc_link_queue_node());
}
size_t calc_size() const {
return impl_.calc_size();
}
private:
friend class MpscLinkQueue;
MpscLinkQueueImpl::Reader impl_;
MpscLinkQueueImpl::Reader &impl() {
return impl_;
}
};
void pop_all(Reader &reader) {
return impl_.pop_all(reader.impl());
}
void pop_all_unsafe(Reader &reader) {
return impl_.pop_all_unsafe(reader.impl());
}
private:
MpscLinkQueueImpl impl_;
};
template <class Value>
class MpscLinkQueueUniquePtrNode {
public:
MpscLinkQueueUniquePtrNode() = default;
explicit MpscLinkQueueUniquePtrNode(unique_ptr<Value> ptr) : ptr_(std::move(ptr)) {
}
MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
return ptr_.release()->to_mpsc_link_queue_node();
}
static MpscLinkQueueUniquePtrNode<Value> from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) {
return MpscLinkQueueUniquePtrNode<Value>(unique_ptr<Value>(Value::from_mpsc_link_queue_node(node)));
}
explicit operator bool() const noexcept {
return ptr_ != nullptr;
}
Value &value() {
return *ptr_;
}
private:
unique_ptr<Value> ptr_;
};
} // namespace td

159
td/tdutils/td/utils/MpscPollableQueue.h Normal file
View File

@@ -0,0 +1,159 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/misc.h"
#include "td/utils/port/EventFd.h"
#if !TD_EVENTFD_UNSUPPORTED
#include "td/utils/port/Mutex.h"
#include <utility>
namespace td {
// interface like in PollableQueue
template <class T>
class MpscPollableQueue {
public:
using ValueType = T;
int reader_wait_nonblock() {
auto ready = reader_vector_.size() - reader_pos_;
if (ready != 0) {
return narrow_cast<int>(ready);
}
for (int i = 0; i < 2; i++) {
auto guard = lock_.lock();
if (writer_vector_.empty()) {
if (i == 1) {
reader_vector_.clear();
reader_pos_ = 0;
wait_event_fd_ = true;
return 0;
}
} else {
reader_vector_.clear();
reader_pos_ = 0;
std::swap(writer_vector_, reader_vector_);
return narrow_cast<int>(reader_vector_.size());
}
event_fd_.acquire();
}
UNREACHABLE();
}
ValueType reader_get_unsafe() {
return std::move(reader_vector_[reader_pos_++]);
}
void reader_flush() {
//nop
}
void writer_put(ValueType value) {
auto guard = lock_.lock();
writer_vector_.push_back(std::move(value));
if (wait_event_fd_) {
wait_event_fd_ = false;
guard.reset();
event_fd_.release();
}
}
EventFd &reader_get_event_fd() {
return event_fd_;
}
void writer_flush() {
//nop
}
bool is_empty() {
auto guard = lock_.lock();
return writer_vector_.empty() && reader_vector_.empty();
}
void init() {
event_fd_.init();
}
void destroy() {
if (!event_fd_.empty()) {
event_fd_.close();
wait_event_fd_ = false;
writer_vector_.clear();
reader_vector_.clear();
reader_pos_ = 0;
}
}
// Just an example of usage
int reader_wait() {
int res;
while ((res = reader_wait_nonblock()) == 0) {
reader_get_event_fd().wait(1000);
}
return res;
}
private:
Mutex lock_;
bool wait_event_fd_{false};
EventFd event_fd_;
std::vector<ValueType> writer_vector_;
std::vector<ValueType> reader_vector_;
size_t reader_pos_{0};
};
} // namespace td
#else
namespace td {
// dummy implementation which shouldn't be used
template <class T>
class MpscPollableQueue {
public:
using ValueType = T;
void init() {
UNREACHABLE();
}
template <class PutValueType>
void writer_put(PutValueType &&value) {
UNREACHABLE();
}
void writer_flush() {
UNREACHABLE();
}
int reader_wait_nonblock() {
UNREACHABLE();
return 0;
}
ValueType reader_get_unsafe() {
UNREACHABLE();
return ValueType();
}
void reader_flush() {
UNREACHABLE();
}
MpscPollableQueue() = default;
MpscPollableQueue(const MpscPollableQueue &) = delete;
MpscPollableQueue &operator=(const MpscPollableQueue &) = delete;
MpscPollableQueue(MpscPollableQueue &&) = delete;
MpscPollableQueue &operator=(MpscPollableQueue &&) = delete;
~MpscPollableQueue() = default;
};
} // namespace td
#endif

27
td/tdutils/td/utils/Named.h Normal file
View File

@@ -0,0 +1,27 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
namespace td {
class Named {
public:
Slice get_name() const {
return name_;
}
void set_name(Slice name) {
name_ = name.str();
}
private:
string name_;
};
} // namespace td

19
td/tdutils/td/utils/NullLog.h Normal file
View File

@@ -0,0 +1,19 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/logging.h"
#include "td/utils/Slice.h"
namespace td {
class NullLog final : public LogInterface {
void do_append(int /*log_level*/, CSlice /*slice*/) final {
}
};
} // namespace td

248
td/tdutils/td/utils/ObjectPool.h Normal file
View File

@@ -0,0 +1,248 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include <atomic>
#include <memory>
#include <utility>
namespace td {
// It is draft object pool implementation
//
// Compared with std::shared_ptr:
// + WeakPtr are much faster. Just pointer copy. No barriers, no atomics.
// - We can't destroy object, because we don't know if it is pointed to by some weak pointer
//
template <class DataT>
class ObjectPool {
struct Storage;
public:
class WeakPtr {
public:
WeakPtr() : generation_(-1), storage_(nullptr) {
}
WeakPtr(int32 generation, Storage *storage) : generation_(generation), storage_(storage) {
}
DataT &operator*() const {
return storage_->data;
}
DataT *operator->() const {
return &**this;
}
// Pattern of usage: 1. Read an object 2. Check if read was valid via is_alive
//
// It is not very usual case of acquire/release use.
// We publish new generation via destruction of the data instead of publishing the object via some flag.
// In usual case if we see a flag, then we are able to use an object.
// In our case if we have used an object and it is already invalid, then generation will mismatch.
bool is_alive() const {
if (!storage_) {
return false;
}
std::atomic_thread_fence(std::memory_order_acquire);
return generation_ == storage_->generation.load(std::memory_order_relaxed);
}
// Used for ActorId
bool is_alive_unsafe() const {
if (!storage_) {
return false;
}
return generation_ == storage_->generation.load(std::memory_order_relaxed);
}
bool empty() const {
return storage_ == nullptr;
}
void clear() {
generation_ = -1;
storage_ = nullptr;
}
int32 generation() {
return generation_;
}
private:
int32 generation_;
Storage *storage_;
};
class OwnerPtr {
public:
OwnerPtr() = default;
OwnerPtr(const OwnerPtr &) = delete;
OwnerPtr &operator=(const OwnerPtr &) = delete;
OwnerPtr(OwnerPtr &&other) noexcept : storage_(other.storage_), parent_(other.parent_) {
other.storage_ = nullptr;
other.parent_ = nullptr;
}
OwnerPtr &operator=(OwnerPtr &&other) noexcept {
if (this != &other) {
storage_ = other.storage_;
parent_ = other.parent_;
other.storage_ = nullptr;
other.parent_ = nullptr;
}
return *this;
}
~OwnerPtr() {
reset();
}
DataT *get() {
return &storage_->data;
}
DataT &operator*() {
return *get();
}
DataT *operator->() {
return get();
}
const DataT *get() const {
return &storage_->data;
}
const DataT &operator*() const {
return *get();
}
const DataT *operator->() const {
return get();
}
WeakPtr get_weak() {
return WeakPtr(storage_->generation.load(std::memory_order_relaxed), storage_);
}
int32 generation() {
return storage_->generation.load(std::memory_order_relaxed);
}
Storage *release() {
auto result = storage_;
storage_ = nullptr;
return result;
}
bool empty() const {
return storage_ == nullptr;
}
void reset() {
if (storage_ != nullptr) {
// for crazy cases when data owns owner pointer to itself.
auto tmp = storage_;
storage_ = nullptr;
parent_->release(OwnerPtr(tmp, parent_));
}
}
private:
friend class ObjectPool;
OwnerPtr(Storage *storage, ObjectPool<DataT> *parent) : storage_(storage), parent_(parent) {
}
Storage *storage_ = nullptr;
ObjectPool<DataT> *parent_ = nullptr;
};
template <class... ArgsT>
OwnerPtr create(ArgsT &&...args) {
Storage *storage = get_storage();
storage->init_data(std::forward<ArgsT>(args)...);
return OwnerPtr(storage, this);
}
OwnerPtr create_empty() {
Storage *storage = get_storage();
return OwnerPtr(storage, this);
}
void set_check_empty(bool flag) {
check_empty_flag_ = flag;
}
void release(OwnerPtr &&owner_ptr) {
Storage *storage = owner_ptr.release();
storage->destroy_data();
release_storage(storage);
}
ObjectPool() = default;
ObjectPool(const ObjectPool &) = delete;
ObjectPool &operator=(const ObjectPool &) = delete;
ObjectPool(ObjectPool &&) = delete;
ObjectPool &operator=(ObjectPool &&) = delete;
~ObjectPool() {
while (head_.load()) {
auto to_delete = head_.load();
head_ = to_delete->next;
delete to_delete;
storage_count_--;
}
LOG_CHECK(storage_count_.load() == 0) << storage_count_.load();
}
private:
struct Storage {
// union {
DataT data;
//};
Storage *next = nullptr;
std::atomic<int32> generation{1};
template <class... ArgsT>
void init_data(ArgsT &&...args) {
// new (&data) DataT(std::forward<ArgsT>(args)...);
data = DataT(std::forward<ArgsT>(args)...);
}
void destroy_data() {
generation.fetch_add(1, std::memory_order_relaxed);
std::atomic_thread_fence(std::memory_order_release);
data.clear();
}
};
std::atomic<int32> storage_count_{0};
std::atomic<Storage *> head_{static_cast<Storage *>(nullptr)};
bool check_empty_flag_ = false;
// TODO(perf): allocation Storages in chunks? Anyway, we won't be able to release them.
// TODO(perf): memory order
// TODO(perf): use another non lockfree list for release on the same thread
// only one thread, so no aba problem
Storage *get_storage() {
if (head_.load() == nullptr) {
storage_count_++;
return new Storage();
}
Storage *res;
while (true) {
res = head_.load();
auto *next = res->next;
if (head_.compare_exchange_weak(res, next)) {
break;
}
}
return res;
}
// release can be called from other thread
void release_storage(Storage *storage) {
while (true) {
auto *save_head = head_.load();
storage->next = save_head;
if (head_.compare_exchange_weak(save_head, storage)) {
break;
}
}
}
};
} // namespace td

41
td/tdutils/td/utils/Observer.h Normal file
View File

@@ -0,0 +1,41 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
namespace td {
class ObserverBase {
public:
ObserverBase() = default;
ObserverBase(const ObserverBase &) = delete;
ObserverBase &operator=(const ObserverBase &) = delete;
ObserverBase(ObserverBase &&) = delete;
ObserverBase &operator=(ObserverBase &&) = delete;
virtual ~ObserverBase() = default;
virtual void notify() = 0;
};
class Observer final : private ObserverBase {
public:
Observer() = default;
explicit Observer(unique_ptr<ObserverBase> &&ptr) : observer_ptr_(std::move(ptr)) {
}
void notify() final {
if (observer_ptr_) {
observer_ptr_->notify();
}
}
private:
unique_ptr<ObserverBase> observer_ptr_;
};
} // namespace td

263
td/tdutils/td/utils/OptionParser.cpp Normal file
View File

@@ -0,0 +1,263 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/OptionParser.h"
#include "td/utils/FlatHashMap.h"
#include "td/utils/logging.h"
#include "td/utils/PathView.h"
#include "td/utils/SliceBuilder.h"
#if TD_PORT_WINDOWS
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
#include "td/utils/port/wstring_convert.h"
#endif
#endif
#if TD_PORT_WINDOWS
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
#include <shellapi.h>
#endif
#endif
namespace td {
void OptionParser::set_usage(Slice executable_name, Slice usage) {
PathView path_view(executable_name);
usage_ = PSTRING() << path_view.file_name() << " " << usage;
}
void OptionParser::set_description(string description) {
description_ = std::move(description);
}
void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
for (auto &option : options_) {
if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) {
LOG(ERROR) << "Ignore duplicate option '" << (short_key == '\0' ? '-' : short_key) << "' '" << long_key << "'";
}
}
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
}
void OptionParser::add_checked_option(char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
}
void OptionParser::add_checked_option(char short_key, Slice long_key, Slice description,
std::function<Status(void)> callback) {
add_option(Option::Type::NoArg, short_key, long_key, description,
[callback = std::move(callback)](Slice) { return callback(); });
}
void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function<void(Slice)> callback) {
add_option(Option::Type::Arg, short_key, long_key, description, [callback = std::move(callback)](Slice parameter) {
callback(parameter);
return Status::OK();
});
}
void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function<void(void)> callback) {
add_option(Option::Type::NoArg, short_key, long_key, description, [callback = std::move(callback)](Slice) {
callback();
return Status::OK();
});
}
void OptionParser::add_check(std::function<Status()> check) {
checks_.push_back(std::move(check));
}
Result<vector<char *>> OptionParser::run(int argc, char *argv[], int expected_non_option_count) {
#if TD_PORT_WINDOWS
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
LPWSTR *utf16_argv = CommandLineToArgvW(GetCommandLineW(), &argc);
if (utf16_argv == nullptr) {
return Status::Error("Failed to parse command line");
}
vector<string> args_storage(argc);
vector<char *> args(argc);
for (int i = 0; i < argc; i++) {
TRY_RESULT_ASSIGN(args_storage[i], from_wstring(utf16_argv[i]));
args[i] = &args_storage[i][0];
}
LocalFree(utf16_argv);
argv = &args[0];
#endif
#endif
return run_impl(argc, argv, expected_non_option_count);
}
Result<vector<char *>> OptionParser::run_impl(int argc, char *argv[], int expected_non_option_count) {
FlatHashMap<char, const Option *> short_options;
FlatHashMap<string, const Option *> long_options;
for (auto &opt : options_) {
if (opt.short_key != '\0') {
short_options[opt.short_key] = &opt;
}
if (!opt.long_key.empty()) {
long_options[opt.long_key] = &opt;
}
}
vector<char *> non_options;
for (int arg_pos = 1; arg_pos < argc; arg_pos++) {
const char *arg = argv[arg_pos];
if (arg[0] != '-' || arg[1] == '\0') {
non_options.push_back(argv[arg_pos]);
continue;
}
if (arg[1] == '-' && arg[2] == '\0') {
// "--"; after it everything is non-option
while (++arg_pos < argc) {
non_options.push_back(argv[arg_pos]);
}
break;
}
if (arg[1] == '-') {
// long option
Slice long_arg(arg + 2);
Slice parameter;
auto equal_pos = long_arg.find('=');
bool has_equal = equal_pos != Slice::npos;
if (has_equal) {
parameter = long_arg.substr(equal_pos + 1);
long_arg = long_arg.substr(0, equal_pos);
}
auto it = long_options.find(long_arg.str());
if (it == long_options.end()) {
return Status::Error(PSLICE() << "Option \"" << long_arg << "\" is unrecognized");
}
auto option = it->second;
switch (option->type) {
case Option::Type::NoArg:
if (has_equal) {
return Status::Error(PSLICE() << "Option \"" << long_arg << "\" must not have an argument");
}
break;
case Option::Type::Arg:
if (!has_equal) {
if (++arg_pos == argc) {
return Status::Error(PSLICE() << "Option \"" << long_arg << "\" requires an argument");
}
parameter = Slice(argv[arg_pos]);
}
break;
default:
UNREACHABLE();
}
TRY_STATUS(option->arg_callback(parameter));
continue;
}
for (size_t opt_pos = 1; arg[opt_pos] != '\0'; opt_pos++) {
auto it = short_options.find(arg[opt_pos]);
if (it == short_options.end()) {
return Status::Error(PSLICE() << "Option \"" << arg[opt_pos] << "\" is unrecognized");
}
auto option = it->second;
Slice parameter;
switch (option->type) {
case Option::Type::NoArg:
// nothing to do
break;
case Option::Type::Arg:
if (arg[opt_pos + 1] == '\0') {
if (++arg_pos == argc) {
return Status::Error(PSLICE() << "Option \"" << arg[opt_pos] << "\" requires an argument");
}
parameter = Slice(argv[arg_pos]);
} else {
parameter = Slice(arg + opt_pos + 1);
opt_pos += parameter.size();
}
break;
default:
UNREACHABLE();
}
TRY_STATUS(option->arg_callback(parameter));
}
}
if (expected_non_option_count >= 0 && non_options.size() != static_cast<size_t>(expected_non_option_count)) {
if (expected_non_option_count == 0) {
return Status::Error("Unexpected non-option parameters specified");
}
if (non_options.size() > static_cast<size_t>(expected_non_option_count)) {
return Status::Error("Too many non-option parameters specified");
} else {
return Status::Error("Too few non-option parameters specified");
}
}
for (auto &check : checks_) {
TRY_STATUS(check());
}
return std::move(non_options);
}
StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o) {
if (!o.usage_.empty()) {
sb << "Usage: " << o.usage_ << "\n\n";
}
if (!o.description_.empty()) {
sb << o.description_ << ". ";
}
sb << "Options:\n";
size_t max_length = 0;
for (auto &opt : o.options_) {
size_t length = 2;
if (!opt.long_key.empty()) {
length += 4 + opt.long_key.size();
}
if (opt.type != OptionParser::Option::Type::NoArg) {
length += 6;
}
if (length > max_length) {
max_length = length;
}
}
max_length++;
for (auto &opt : o.options_) {
bool has_short_key = opt.short_key != '\0';
sb << " ";
size_t length = max_length;
if (has_short_key) {
sb << '-' << opt.short_key;
} else {
sb << " ";
}
length -= 2;
if (!opt.long_key.empty()) {
if (has_short_key) {
sb << ", ";
} else {
sb << " ";
}
sb << "--" << opt.long_key;
length -= 4 + opt.long_key.size();
}
if (opt.type != OptionParser::Option::Type::NoArg) {
sb << "=<arg>";
length -= 6;
}
sb << string(length, ' ') << opt.description;
sb << '\n';
}
return sb;
}
} // namespace td

81
td/tdutils/td/utils/OptionParser.h Normal file
View File

@@ -0,0 +1,81 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
#include <functional>
namespace td {
class OptionParser {
class Option {
public:
enum class Type { NoArg, Arg };
Type type;
char short_key;
string long_key;
string description;
std::function<Status(Slice)> arg_callback;
};
void add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback);
public:
template <class T>
static std::function<Status(Slice)> parse_integer(T &value) {
return [&value](Slice value_str) {
TRY_RESULT_ASSIGN(value, to_integer_safe<T>(value_str));
return Status::OK();
};
}
static std::function<void(Slice)> parse_string(string &value) {
return [&value](Slice value_str) {
value = value_str.str();
};
}
void set_usage(Slice executable_name, Slice usage);
void set_description(string description);
void add_checked_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback);
void add_checked_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback) = delete;
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback) = delete;
void add_option(char short_key, Slice long_key, Slice description, std::function<void(Slice)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<void(void)> callback);
void add_check(std::function<Status()> check);
// returns found non-option parameters
Result<vector<char *>> run(int argc, char *argv[], int expected_non_option_count = -1) TD_WARN_UNUSED_RESULT;
// for testing only
Result<vector<char *>> run_impl(int argc, char *argv[], int expected_non_option_count) TD_WARN_UNUSED_RESULT;
friend StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o);
private:
vector<Option> options_;
vector<std::function<Status()>> checks_;
string usage_;
string description_;
};
} // namespace td

99
td/tdutils/td/utils/OrderedEventsProcessor.h Normal file
View File

@@ -0,0 +1,99 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include <utility>
namespace td {
// Process states in order defined by their SeqNo
template <class DataT>
class OrderedEventsProcessor {
public:
using SeqNo = uint64;
OrderedEventsProcessor() = default;
explicit OrderedEventsProcessor(SeqNo offset) : offset_(offset), begin_(offset_), end_(offset_) {
}
template <class FunctionT>
void clear(FunctionT &&function) {
for (auto &it : data_array_) {
if (it.second) {
function(std::move(it.first));
}
}
*this = OrderedEventsProcessor();
}
void clear() {
*this = OrderedEventsProcessor();
}
template <class FromDataT, class FunctionT>
void add(SeqNo seq_no, FromDataT &&data, FunctionT &&function) {
LOG_CHECK(seq_no >= begin_) << seq_no << ">=" << begin_; // or ignore?
if (seq_no == begin_) { // run now
begin_++;
function(seq_no, std::forward<FromDataT>(data));
while (begin_ < end_) {
auto &data_flag = data_array_[static_cast<size_t>(begin_ - offset_)];
if (!data_flag.second) {
break;
}
function(begin_, std::move(data_flag.first));
data_flag.second = false;
begin_++;
}
if (begin_ > end_) {
end_ = begin_;
}
if (begin_ == end_) {
offset_ = begin_;
}
// try_compactify
auto begin_pos = static_cast<size_t>(begin_ - offset_);
if (begin_pos > 5 && begin_pos * 2 > data_array_.size()) {
data_array_.erase(data_array_.begin(), data_array_.begin() + begin_pos);
offset_ = begin_;
}
} else {
auto pos = static_cast<size_t>(seq_no - offset_);
auto need_size = pos + 1;
if (data_array_.size() < need_size) {
data_array_.resize(need_size);
}
data_array_[pos].first = std::forward<FromDataT>(data);
data_array_[pos].second = true;
if (end_ < seq_no + 1) {
end_ = seq_no + 1;
}
}
}
bool has_events() const {
return begin_ != end_;
}
SeqNo max_unfinished_seq_no() {
return end_ - 1;
}
SeqNo max_finished_seq_no() {
return begin_ - 1;
}
private:
SeqNo offset_ = 1;
SeqNo begin_ = 1;
SeqNo end_ = 1;
std::vector<std::pair<DataT, bool>> data_array_;
};
} // namespace td

189
td/tdutils/td/utils/Parser.h Normal file
View File

@@ -0,0 +1,189 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/SliceBuilder.h"
#include "td/utils/Status.h"
#include <cstring>
#include <utility>
namespace td {
namespace detail {
template <class SliceT>
class ParserImpl {
public:
explicit ParserImpl(SliceT data) : ptr_(data.begin()), end_(data.end()), status_() {
}
ParserImpl(ParserImpl &&other) noexcept : ptr_(other.ptr_), end_(other.end_), status_(std::move(other.status_)) {
other.clear();
}
ParserImpl &operator=(ParserImpl &&other) noexcept {
if (&other == this) {
return *this;
}
ptr_ = other.ptr_;
end_ = other.end_;
status_ = std::move(other.status_);
other.clear();
return *this;
}
ParserImpl(const ParserImpl &) = delete;
ParserImpl &operator=(const ParserImpl &) = delete;
~ParserImpl() = default;
bool empty() const {
return ptr_ == end_;
}
void clear() {
ptr_ = SliceT().begin();
end_ = ptr_;
status_ = Status::OK();
}
SliceT read_till_nofail(char c) {
if (status_.is_error()) {
return SliceT();
}
auto till = static_cast<decltype(ptr_)>(std::memchr(ptr_, c, end_ - ptr_));
if (till == nullptr) {
till = end_;
}
SliceT result(ptr_, till);
ptr_ = till;
return result;
}
SliceT read_till_nofail(Slice str) {
if (status_.is_error()) {
return SliceT();
}
auto best_till = end_;
for (auto c : str) {
auto till = static_cast<decltype(ptr_)>(std::memchr(ptr_, c, end_ - ptr_));
if (till != nullptr && till < best_till) {
best_till = till;
}
}
SliceT result(ptr_, best_till);
ptr_ = best_till;
return result;
}
template <class F>
SliceT read_while(const F &f) {
auto save_ptr = ptr_;
while (ptr_ != end_ && f(*ptr_)) {
ptr_++;
}
return SliceT(save_ptr, ptr_);
}
SliceT read_all() {
auto save_ptr = ptr_;
ptr_ = end_;
return SliceT(save_ptr, ptr_);
}
SliceT read_till(char c) {
if (status_.is_error()) {
return SliceT();
}
SliceT res = read_till_nofail(c);
if (ptr_ == end_ || ptr_[0] != c) {
status_ = Status::Error(PSLICE() << "Read till '" << c << "' failed");
return SliceT();
}
return res;
}
char peek_char() {
if (ptr_ == end_) {
return 0;
}
return *ptr_;
}
char *ptr() {
return ptr_;
}
void skip_nofail(char c) {
if (ptr_ != end_ && ptr_[0] == c) {
ptr_++;
}
}
void skip(char c) {
if (status_.is_error()) {
return;
}
if (ptr_ == end_ || ptr_[0] != c) {
status_ = Status::Error(PSLICE() << "Skip '" << c << "' failed");
return;
}
ptr_++;
}
bool try_skip(char c) {
if (ptr_ != end_ && ptr_[0] == c) {
ptr_++;
return true;
}
return false;
}
bool try_skip(Slice prefix) {
if (prefix.size() > static_cast<size_t>(end_ - ptr_) || prefix != Slice(ptr_, prefix.size())) {
return false;
}
advance(prefix.size());
return true;
}
void skip_till_not(Slice str) {
while (ptr_ != end_) {
if (std::memchr(str.data(), *ptr_, str.size()) == nullptr) {
break;
}
ptr_++;
}
}
void skip_whitespaces() {
skip_till_not(" \t\r\n");
}
SliceT read_word() {
skip_whitespaces();
return read_till_nofail(" \t\r\n");
}
SliceT data() const {
return SliceT(ptr_, end_);
}
Status &status() {
return status_;
}
void advance(size_t diff) {
ptr_ += diff;
CHECK(ptr_ <= end_);
}
private:
decltype(std::declval<SliceT>().begin()) ptr_;
decltype(std::declval<SliceT>().end()) end_;
Status status_;
};
} // namespace detail
using Parser = detail::ParserImpl<MutableSlice>;
using ConstParser = detail::ParserImpl<Slice>;
} // namespace td

70
td/tdutils/td/utils/PathView.cpp Normal file
View File

@@ -0,0 +1,70 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/PathView.h"
#include "td/utils/common.h"
#include "td/utils/misc.h"
namespace td {
PathView::PathView(Slice path) : path_(path) {
last_slash_ = narrow_cast<int32>(path_.size()) - 1;
while (last_slash_ >= 0 && !is_slash(path_[last_slash_])) {
last_slash_--;
}
last_dot_ = static_cast<int32>(path_.size());
for (auto i = last_dot_ - 1; i > last_slash_ + 1; i--) {
if (path_[i] == '.') {
last_dot_ = i;
break;
}
}
}
Slice PathView::parent_dir_noslash() const {
if (last_slash_ < 0) {
return Slice(".");
}
if (last_slash_ == 0) {
static char buf[1];
buf[0] = TD_DIR_SLASH;
return Slice(buf, 1);
}
return path_.substr(0, last_slash_);
}
Slice PathView::relative(Slice path, Slice dir, bool force) {
if (begins_with(path, dir)) {
path.remove_prefix(dir.size());
return path;
}
if (force) {
return Slice();
}
return path;
}
Slice PathView::dir_and_file(Slice path) {
auto last_slash = static_cast<int32>(path.size()) - 1;
while (last_slash >= 0 && !is_slash(path[last_slash])) {
last_slash--;
}
if (last_slash < 0) {
return Slice();
}
last_slash--;
while (last_slash >= 0 && !is_slash(path[last_slash])) {
last_slash--;
}
if (last_slash < 0) {
return Slice();
}
return path.substr(last_slash + 1);
}
} // namespace td

81
td/tdutils/td/utils/PathView.h Normal file
View File

@@ -0,0 +1,81 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/Slice.h"
namespace td {
class PathView {
public:
explicit PathView(Slice path);
bool empty() const {
return path_.empty();
}
bool is_dir() const {
if (empty()) {
return false;
}
return is_slash(path_.back());
}
Slice parent_dir() const {
return path_.substr(0, last_slash_ + 1);
}
Slice parent_dir_noslash() const;
Slice extension() const {
if (last_dot_ == static_cast<int32>(path_.size())) {
return Slice();
}
return path_.substr(last_dot_ + 1);
}
Slice without_extension() const {
return path_.substr(0, last_dot_);
}
Slice file_stem() const {
return path_.substr(last_slash_ + 1, last_dot_ - last_slash_ - 1);
}
Slice file_name() const {
return path_.substr(last_slash_ + 1);
}
Slice file_name_without_extension() const {
return path_.substr(last_slash_ + 1, last_dot_ - last_slash_ - 1);
}
Slice path() const {
return path_;
}
bool is_absolute() const {
return !empty() && (is_slash(path_[0]) || (path_.size() >= 3 && path_[1] == ':' && is_slash(path_[2])));
}
bool is_relative() const {
return !is_absolute();
}
static Slice relative(Slice path, Slice dir, bool force = false);
static Slice dir_and_file(Slice path);
private:
static bool is_slash(char c) {
return c == '/' || c == '\\';
}
Slice path_;
int32 last_slash_;
int32 last_dot_;
};
} // namespace td

373
td/tdutils/td/utils/Promise.h Normal file
View File

@@ -0,0 +1,373 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/CancellationToken.h"
#include "td/utils/common.h"
#include "td/utils/invoke.h"
#include "td/utils/MovableValue.h"
#include "td/utils/Status.h"
#include <tuple>
#include <type_traits>
#include <utility>
namespace td {
template <class T = Unit>
class PromiseInterface {
public:
PromiseInterface() = default;
PromiseInterface(const PromiseInterface &) = delete;
PromiseInterface &operator=(const PromiseInterface &) = delete;
PromiseInterface(PromiseInterface &&) = default;
PromiseInterface &operator=(PromiseInterface &&) = default;
virtual ~PromiseInterface() = default;
virtual void set_value(T &&value) {
set_result(std::move(value));
}
virtual void set_error(Status &&error) {
set_result(std::move(error));
}
virtual void set_result(Result<T> &&result) {
if (result.is_ok()) {
set_value(result.move_as_ok());
} else {
set_error(result.move_as_error());
}
}
virtual bool is_cancellable() const {
return false;
}
virtual bool is_canceled() const {
return false;
}
};
template <class T>
class SafePromise;
template <class T = Unit>
class Promise;
namespace detail {
template <typename T>
struct GetArg final : public GetArg<decltype(&T::operator())> {};
template <class C, class R, class Arg>
class GetArg<R (C::*)(Arg)> {
public:
using type = Arg;
};
template <class C, class R, class Arg>
class GetArg<R (C::*)(Arg) const> {
public:
using type = Arg;
};
template <class T>
using get_arg_t = std::decay_t<typename GetArg<T>::type>;
template <class T>
struct DropResult {
using type = T;
};
template <class T>
struct DropResult<Result<T>> {
using type = T;
};
template <class T>
using drop_result_t = typename DropResult<T>::type;
template <class ValueT, class FunctionT>
class LambdaPromise : public PromiseInterface<ValueT> {
enum class State : int32 { Empty, Ready, Complete };
public:
void set_value(ValueT &&value) override {
CHECK(state_.get() == State::Ready);
do_ok(std::move(value));
state_ = State::Complete;
}
void set_error(Status &&error) override {
if (state_.get() == State::Ready) {
do_error(std::move(error));
state_ = State::Complete;
}
}
LambdaPromise(const LambdaPromise &) = delete;
LambdaPromise &operator=(const LambdaPromise &) = delete;
LambdaPromise(LambdaPromise &&) = default;
LambdaPromise &operator=(LambdaPromise &&) = default;
~LambdaPromise() override {
if (state_.get() == State::Ready) {
do_error(Status::Error("Lost promise"));
}
}
template <class FromT>
LambdaPromise(FromT &&func) : func_(std::forward<FromT>(func)), state_(State::Ready) {
}
private:
FunctionT func_;
MovableValue<State> state_{State::Empty};
template <class F = FunctionT>
std::enable_if_t<is_callable<F, Result<ValueT>>::value, void> do_error(Status &&status) {
func_(Result<ValueT>(std::move(status)));
}
template <class Y, class F = FunctionT>
std::enable_if_t<!is_callable<F, Result<ValueT>>::value, void> do_error(Y &&status) {
func_(Auto());
}
template <class F = FunctionT>
std::enable_if_t<is_callable<F, Result<ValueT>>::value, void> do_ok(ValueT &&value) {
func_(Result<ValueT>(std::move(value)));
}
template <class F = FunctionT>
std::enable_if_t<!is_callable<F, Result<ValueT>>::value, void> do_ok(ValueT &&value) {
func_(std::move(value));
}
};
template <class T>
struct is_promise_interface : std::false_type {};
template <class U>
struct is_promise_interface<PromiseInterface<U>> : std::true_type {};
template <class U>
struct is_promise_interface<Promise<U>> : std::true_type {};
template <class T>
struct is_promise_interface_ptr : std::false_type {};
template <class U>
struct is_promise_interface_ptr<unique_ptr<U>> : std::true_type {};
template <class T = void, class F = void, std::enable_if_t<std::is_same<T, void>::value, bool> has_t = false>
auto lambda_promise(F &&f) {
return LambdaPromise<drop_result_t<get_arg_t<std::decay_t<F>>>, std::decay_t<F>>(std::forward<F>(f));
}
template <class T = void, class F = void, std::enable_if_t<!std::is_same<T, void>::value, bool> has_t = true>
auto lambda_promise(F &&f) {
return LambdaPromise<T, std::decay_t<F>>(std::forward<F>(f));
}
template <class T, class F,
std::enable_if_t<is_promise_interface<std::decay_t<F>>::value, bool> from_promise_interface = true>
auto &&promise_interface(F &&f) {
return std::forward<F>(f);
}
template <class T, class F,
std::enable_if_t<!is_promise_interface<std::decay_t<F>>::value, bool> from_promise_interface = false>
auto promise_interface(F &&f) {
return lambda_promise<T>(std::forward<F>(f));
}
template <class T, class F,
std::enable_if_t<is_promise_interface_ptr<std::decay_t<F>>::value, bool> from_promise_interface = true>
auto promise_interface_ptr(F &&f) {
return std::forward<F>(f);
}
template <class T, class F,
std::enable_if_t<!is_promise_interface_ptr<std::decay_t<F>>::value, bool> from_promise_interface = false>
auto promise_interface_ptr(F &&f) {
return td::make_unique<std::decay_t<decltype(promise_interface<T>(std::forward<F>(f)))>>(
promise_interface<T>(std::forward<F>(f)));
}
} // namespace detail
template <class T>
class Promise {
public:
void set_value(T &&value) {
if (!promise_) {
return;
}
promise_->set_value(std::move(value));
promise_.reset();
}
void set_error(Status &&error) {
if (!promise_) {
return;
}
promise_->set_error(std::move(error));
promise_.reset();
}
void set_result(Result<T> &&result) {
if (!promise_) {
return;
}
promise_->set_result(std::move(result));
promise_.reset();
}
void reset() {
promise_.reset();
}
bool is_cancellable() const {
if (!promise_) {
return false;
}
return promise_->is_cancellable();
}
bool is_canceled() const {
if (!promise_) {
return false;
}
return promise_->is_canceled();
}
unique_ptr<PromiseInterface<T>> release() {
return std::move(promise_);
}
Promise() = default;
explicit Promise(unique_ptr<PromiseInterface<T>> promise) : promise_(std::move(promise)) {
}
Promise(Auto) {
}
Promise(SafePromise<T> &&other);
Promise &operator=(SafePromise<T> &&other);
template <class F, std::enable_if_t<!std::is_same<std::decay_t<F>, Promise>::value, int> = 0>
Promise(F &&f) : promise_(detail::promise_interface_ptr<T>(std::forward<F>(f))) {
}
explicit operator bool() const noexcept {
return static_cast<bool>(promise_);
}
private:
unique_ptr<PromiseInterface<T>> promise_;
};
template <class T = Unit>
class SafePromise {
public:
SafePromise(Promise<T> promise, Result<T> result) : promise_(std::move(promise)), result_(std::move(result)) {
}
SafePromise(const SafePromise &) = delete;
SafePromise &operator=(const SafePromise &) = delete;
SafePromise(SafePromise &&) = default;
SafePromise &operator=(SafePromise &&) = default;
~SafePromise() {
if (promise_) {
promise_.set_result(std::move(result_));
}
}
Promise<T> release() {
return std::move(promise_);
}
private:
Promise<T> promise_;
Result<T> result_;
};
template <class T>
Promise<T>::Promise(SafePromise<T> &&other) : Promise(other.release()) {
}
template <class T>
Promise<T> &Promise<T>::operator=(SafePromise<T> &&other) {
*this = other.release();
return *this;
}
namespace detail {
template <class PromiseT>
class CancellablePromise final : public PromiseT {
public:
template <class... ArgsT>
CancellablePromise(CancellationToken cancellation_token, ArgsT &&...args)
: PromiseT(std::forward<ArgsT>(args)...), cancellation_token_(std::move(cancellation_token)) {
}
bool is_cancellable() const final {
return true;
}
bool is_canceled() const final {
return static_cast<bool>(cancellation_token_);
}
private:
CancellationToken cancellation_token_;
};
template <class... ArgsT>
class JoinPromise final : public PromiseInterface<Unit> {
public:
explicit JoinPromise(ArgsT &&...arg) : promises_(std::forward<ArgsT>(arg)...) {
}
void set_value(Unit &&) final {
tuple_for_each(promises_, [](auto &promise) { promise.set_value(Unit()); });
}
void set_error(Status &&error) final {
tuple_for_each(promises_, [&error](auto &promise) { promise.set_error(error.clone()); });
}
private:
std::tuple<std::decay_t<ArgsT>...> promises_;
};
} // namespace detail
class PromiseCreator {
public:
template <class OkT, class ArgT = detail::drop_result_t<detail::get_arg_t<OkT>>>
static Promise<ArgT> lambda(OkT &&ok) {
return Promise<ArgT>(td::make_unique<detail::LambdaPromise<ArgT, std::decay_t<OkT>>>(std::forward<OkT>(ok)));
}
template <class OkT, class ArgT = detail::drop_result_t<detail::get_arg_t<OkT>>>
static auto cancellable_lambda(CancellationToken cancellation_token, OkT &&ok) {
return Promise<ArgT>(td::make_unique<detail::CancellablePromise<detail::LambdaPromise<ArgT, std::decay_t<OkT>>>>(
std::move(cancellation_token), std::forward<OkT>(ok)));
}
template <class... ArgsT>
static Promise<> join(ArgsT &&...args) {
return Promise<>(td::make_unique<detail::JoinPromise<ArgsT...>>(std::forward<ArgsT>(args)...));
}
};
inline void set_promises(vector<Promise<Unit>> &promises) {
auto moved_promises = std::move(promises);
promises.clear();
for (auto &promise : moved_promises) {
promise.set_value(Unit());
}
}
template <class T>
void fail_promises(vector<Promise<T>> &promises, Status &&error) {
CHECK(error.is_error());
auto moved_promises = std::move(promises);
promises.clear();
auto size = moved_promises.size();
if (size == 0) {
return;
}
size--;
for (size_t i = 0; i < size; i++) {
auto &promise = moved_promises[i];
if (promise) {
promise.set_error(error.clone());
}
}
moved_promises[size].set_error(std::move(error));
}
} // namespace td

203
td/tdutils/td/utils/Random.cpp Normal file
View File

@@ -0,0 +1,203 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/Random.h"
#include "td/utils/logging.h"
#include "td/utils/port/thread_local.h"
#if TD_HAVE_OPENSSL
#include <openssl/rand.h>
#endif
#include <atomic>
#include <cstring>
#include <limits>
#include <random>
namespace td {
#if TD_HAVE_OPENSSL
namespace {
std::atomic<int64> random_seed_generation{0};
} // namespace
void Random::secure_bytes(MutableSlice dest) {
Random::secure_bytes(dest.ubegin(), dest.size());
}
void Random::secure_bytes(unsigned char *ptr, size_t size) {
constexpr size_t BUF_SIZE = 512;
static TD_THREAD_LOCAL unsigned char *buf; // static zero-initialized
static TD_THREAD_LOCAL size_t buf_pos;
static TD_THREAD_LOCAL int64 generation;
if (init_thread_local<unsigned char[]>(buf, BUF_SIZE)) {
buf_pos = BUF_SIZE;
generation = 0;
}
if (ptr == nullptr) {
MutableSlice(buf, BUF_SIZE).fill_zero_secure();
buf_pos = BUF_SIZE;
return;
}
if (generation != random_seed_generation.load(std::memory_order_relaxed)) {
generation = random_seed_generation.load(std::memory_order_acquire);
buf_pos = BUF_SIZE;
}
auto ready = min(size, BUF_SIZE - buf_pos);
if (ready != 0) {
std::memcpy(ptr, buf + buf_pos, ready);
buf_pos += ready;
ptr += ready;
size -= ready;
if (size == 0) {
return;
}
}
if (size < BUF_SIZE) {
int err = RAND_bytes(buf, static_cast<int>(BUF_SIZE));
// TODO: it CAN fail
LOG_IF(FATAL, err != 1);
buf_pos = size;
std::memcpy(ptr, buf, size);
return;
}
CHECK(size <= static_cast<size_t>(std::numeric_limits<int>::max()));
int err = RAND_bytes(ptr, static_cast<int>(size));
// TODO: it CAN fail
LOG_IF(FATAL, err != 1);
}
int32 Random::secure_int32() {
int32 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(int32));
return res;
}
int64 Random::secure_int64() {
int64 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(int64));
return res;
}
uint32 Random::secure_uint32() {
uint32 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(uint32));
return res;
}
uint64 Random::secure_uint64() {
uint64 res = 0;
secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(uint64));
return res;
}
void Random::add_seed(Slice bytes, double entropy) {
RAND_add(bytes.data(), static_cast<int>(bytes.size()), entropy);
random_seed_generation++;
}
void Random::secure_cleanup() {
Random::secure_bytes(nullptr, 0);
}
#endif
static unsigned int rand_device_helper() {
static TD_THREAD_LOCAL std::random_device *rd;
init_thread_local<std::random_device>(rd);
return (*rd)();
}
uint32 Random::fast_uint32() {
static TD_THREAD_LOCAL std::mt19937 *gen;
if (!gen) {
auto &rg = rand_device_helper;
std::seed_seq seq{rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg()};
init_thread_local<std::mt19937>(gen, seq);
}
return static_cast<uint32>((*gen)());
}
uint64 Random::fast_uint64() {
static TD_THREAD_LOCAL std::mt19937_64 *gen;
if (!gen) {
auto &rg = rand_device_helper;
std::seed_seq seq{rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg()};
init_thread_local<std::mt19937_64>(gen, seq);
}
return static_cast<uint64>((*gen)());
}
int Random::fast(int min_value, int max_value) {
if (min_value == std::numeric_limits<int>::min() && max_value == std::numeric_limits<int>::max()) {
// to prevent integer overflow and division by zero
min_value++;
}
DCHECK(min_value <= max_value);
return static_cast<int>(min_value + fast_uint32() % (max_value - min_value + 1)); // TODO signed_cast
}
double Random::fast(double min_value, double max_value) {
DCHECK(min_value <= max_value);
return min_value + fast_uint32() * 1.0 / std::numeric_limits<uint32>::max() * (max_value - min_value);
}
bool Random::fast_bool() {
return (fast_uint32() & 1) != 0;
}
Random::Xorshift128plus::Xorshift128plus(uint64 seed) {
auto next = [&] {
// splitmix64
seed += static_cast<uint64>(0x9E3779B97F4A7C15ull);
uint64 z = seed;
z = (z ^ (z >> 30)) * static_cast<uint64>(0xBF58476D1CE4E5B9ull);
z = (z ^ (z >> 27)) * static_cast<uint64>(0x94D049BB133111EBull);
return z ^ (z >> 31);
};
seed_[0] = next();
seed_[1] = next();
}
Random::Xorshift128plus::Xorshift128plus(uint64 seed_a, uint64 seed_b) {
seed_[0] = seed_a;
seed_[1] = seed_b;
}
uint64 Random::Xorshift128plus::operator()() {
uint64 x = seed_[0];
const uint64 y = seed_[1];
seed_[0] = y;
x ^= x << 23;
seed_[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
return seed_[1] + y;
}
int Random::Xorshift128plus::fast(int min_value, int max_value) {
return static_cast<int>((*this)() % (max_value - min_value + 1) + min_value);
}
int64 Random::Xorshift128plus::fast64(int64 min_value, int64 max_value) {
return static_cast<int64>((*this)() % (max_value - min_value + 1) + min_value);
}
void Random::Xorshift128plus::bytes(MutableSlice dest) {
int cnt = 0;
uint64 buf = 0;
for (auto &c : dest) {
if (cnt == 0) {
buf = operator()();
cnt = 8;
}
cnt--;
c = static_cast<char>(buf & 255);
buf >>= 8;
}
}
} // namespace td

68
td/tdutils/td/utils/Random.h Normal file
View File

@@ -0,0 +1,68 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include <utility>
namespace td {
class Random {
public:
#if TD_HAVE_OPENSSL
static void secure_bytes(MutableSlice dest);
static void secure_bytes(unsigned char *ptr, size_t size);
static int32 secure_int32();
static int64 secure_int64();
static uint32 secure_uint32();
static uint64 secure_uint64();
// works only for current thread
static void add_seed(Slice bytes, double entropy = 0);
static void secure_cleanup();
template <class T>
static void shuffle(vector<T> &v) {
for (size_t i = 1; i < v.size(); i++) {
auto pos = static_cast<size_t>(secure_int32()) % (i + 1);
using std::swap;
swap(v[i], v[pos]);
}
}
#endif
static uint32 fast_uint32();
static uint64 fast_uint64();
// distribution is not uniform, min_value and max_value are included
static int fast(int min_value, int max_value);
static double fast(double min_value, double max_value);
static bool fast_bool();
class Fast {
public:
uint64 operator()() {
return fast_uint64();
}
};
class Xorshift128plus {
public:
explicit Xorshift128plus(uint64 seed);
Xorshift128plus(uint64 seed_a, uint64 seed_b);
uint64 operator()();
int fast(int min_value, int max_value);
int64 fast64(int64 min_value, int64 max_value);
void bytes(MutableSlice dest);
private:
uint64 seed_[2];
};
};
} // namespace td

77
td/tdutils/td/utils/ScopeGuard.h Normal file
View File

@@ -0,0 +1,77 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <cstdlib>
#include <memory>
#include <type_traits>
#include <utility>
namespace td {
class Guard {
public:
Guard() = default;
Guard(const Guard &) = delete;
Guard &operator=(const Guard &) = delete;
Guard(Guard &&) = default;
Guard &operator=(Guard &&) = default;
virtual ~Guard() = default;
virtual void dismiss() {
std::abort();
}
};
template <class FunctionT>
class LambdaGuard final : public Guard {
public:
explicit LambdaGuard(const FunctionT &func) : func_(func) {
}
explicit LambdaGuard(FunctionT &&func) : func_(std::move(func)) {
}
LambdaGuard(const LambdaGuard &) = delete;
LambdaGuard &operator=(const LambdaGuard &) = delete;
LambdaGuard(LambdaGuard &&other) : func_(std::move(other.func_)), dismissed_(other.dismissed_) {
other.dismissed_ = true;
}
LambdaGuard &operator=(LambdaGuard &&) = delete;
void dismiss() final {
dismissed_ = true;
}
~LambdaGuard() final {
if (!dismissed_) {
func_();
}
}
private:
FunctionT func_;
bool dismissed_ = false;
};
template <class F>
unique_ptr<Guard> create_lambda_guard(F &&f) {
return make_unique<LambdaGuard<F>>(std::forward<F>(f));
}
template <class F>
std::shared_ptr<Guard> create_shared_lambda_guard(F &&f) {
return std::make_shared<LambdaGuard<F>>(std::forward<F>(f));
}
enum class ScopeExit {};
template <class FunctionT>
auto operator+(ScopeExit, FunctionT &&func) {
return LambdaGuard<std::decay_t<FunctionT>>(std::forward<FunctionT>(func));
}
} // namespace td
#define SCOPE_EXIT auto TD_CONCAT(SCOPE_EXIT_VAR_, __LINE__) = ::td::ScopeExit() + [&]

129
td/tdutils/td/utils/SetNode.h Normal file
View File

@@ -0,0 +1,129 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/HashTableUtils.h"
#include <type_traits>
#include <utility>
namespace td {
template <class KeyT, class EqT, class Enable = void>
struct SetNode {
using public_key_type = KeyT;
using public_type = const KeyT;
using second_type = KeyT; // TODO: remove second_type?
KeyT first;
const KeyT &key() const {
return first;
}
const KeyT &get_public() {
return first;
}
SetNode() : first() {
}
explicit SetNode(KeyT key) : first(std::move(key)) {
}
SetNode(const SetNode &) = delete;
SetNode &operator=(const SetNode &) = delete;
SetNode(SetNode &&other) noexcept {
*this = std::move(other);
}
void operator=(SetNode &&other) noexcept {
DCHECK(empty());
DCHECK(!other.empty());
first = std::move(other.first);
other.first = KeyT();
}
~SetNode() = default;
void copy_from(const SetNode &other) {
DCHECK(empty());
first = other.first;
DCHECK(!empty());
}
bool empty() const {
return is_hash_table_key_empty<EqT>(first);
}
void clear() {
first = KeyT();
DCHECK(empty());
}
void emplace(KeyT key) {
first = std::move(key);
}
};
template <class KeyT, class EqT>
struct SetNode<KeyT, EqT, typename std::enable_if_t<(sizeof(KeyT) > 28 * sizeof(void *))>> {
struct Impl {
using second_type = KeyT;
KeyT first;
template <class InputKeyT>
explicit Impl(InputKeyT &&key) : first(std::forward<InputKeyT>(key)) {
DCHECK(!is_hash_table_key_empty<EqT>(first));
}
Impl(const Impl &) = delete;
Impl &operator=(const Impl &) = delete;
Impl(Impl &&) = delete;
Impl &operator=(Impl &&) = delete;
};
using public_key_type = KeyT;
using public_type = const KeyT;
using second_type = KeyT; // TODO: remove second_type?
unique_ptr<Impl> impl_;
const KeyT &key() const {
DCHECK(!empty());
return impl_->first;
}
const KeyT &get_public() {
DCHECK(!empty());
return impl_->first;
}
SetNode() : impl_() {
}
explicit SetNode(KeyT key) : impl_(td::make_unique<Impl>(std::move(key))) {
}
void copy_from(const SetNode &other) {
DCHECK(empty());
impl_ = td::make_unique<Impl>(other.impl_->first);
DCHECK(!empty());
}
bool empty() const {
return impl_ == nullptr;
}
void clear() {
DCHECK(!empty());
impl_ = nullptr;
}
void emplace(KeyT key) {
DCHECK(empty());
impl_ = td::make_unique<Impl>(std::move(key));
}
};
} // namespace td

288
td/tdutils/td/utils/SharedObjectPool.h Normal file
View File

@@ -0,0 +1,288 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/MpscLinkQueue.h"
#include <atomic>
#include <memory>
#include <new>
#include <utility>
namespace td {
namespace detail {
class AtomicRefCnt {
public:
explicit AtomicRefCnt(uint64 cnt) : cnt_(cnt) {
}
void inc() {
cnt_.fetch_add(1, std::memory_order_relaxed);
}
bool dec() {
return cnt_.fetch_sub(1, std::memory_order_acq_rel) == 1;
}
uint64 value() const {
return cnt_.load(std::memory_order_relaxed);
}
private:
std::atomic<uint64> cnt_{0};
};
template <class DataT, class DeleterT>
class SharedPtrRaw
: public DeleterT
, private MpscLinkQueueImpl::Node {
public:
explicit SharedPtrRaw(DeleterT deleter) : DeleterT(std::move(deleter)), ref_cnt_{0}, option_magic_(Magic) {
}
~SharedPtrRaw() {
CHECK(use_cnt() == 0);
CHECK(option_magic_ == Magic);
}
template <class... ArgsT>
void init_data(ArgsT &&...args) {
new (&option_data_) DataT(std::forward<ArgsT>(args)...);
}
void destroy_data() {
option_data_.~DataT();
option_magic_ = Magic;
}
uint64 use_cnt() const {
return ref_cnt_.value();
}
void inc() {
ref_cnt_.inc();
}
bool dec() {
return ref_cnt_.dec();
}
DataT &data() {
return option_data_;
}
static SharedPtrRaw *from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) {
return static_cast<SharedPtrRaw<DataT, DeleterT> *>(node);
}
MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
return static_cast<MpscLinkQueueImpl::Node *>(this);
}
private:
AtomicRefCnt ref_cnt_;
enum { Magic = 0x732817a2 };
union {
DataT option_data_;
uint32 option_magic_;
};
};
template <class T, class DeleterT = std::default_delete<T>>
class SharedPtr {
public:
using Raw = detail::SharedPtrRaw<T, DeleterT>;
struct acquire_t {};
SharedPtr() = default;
~SharedPtr() {
if (!raw_) {
return;
}
reset();
}
explicit SharedPtr(Raw *raw) : raw_(raw) {
if (raw_) {
raw_->inc();
}
}
SharedPtr(acquire_t, Raw *raw) : raw_(raw) {
}
SharedPtr(const SharedPtr &other) : SharedPtr(other.raw_) {
}
SharedPtr &operator=(const SharedPtr &other) {
if (this == &other) {
return *this;
}
if (other.raw_) {
other.raw_->inc();
}
reset(other.raw_);
return *this;
}
SharedPtr(SharedPtr &&other) noexcept : raw_(other.raw_) {
other.raw_ = nullptr;
}
SharedPtr &operator=(SharedPtr &&other) noexcept {
reset(other.raw_);
other.raw_ = nullptr;
return *this;
}
bool empty() const noexcept {
return raw_ == nullptr;
}
explicit operator bool() const noexcept {
return !empty();
}
uint64 use_cnt() const {
if (!raw_) {
return 0;
}
return raw_->use_cnt();
}
T &operator*() const {
return raw_->data();
}
T *operator->() const {
return &raw_->data();
}
Raw *release() {
auto res = raw_;
raw_ = nullptr;
return res;
}
void reset(Raw *new_raw = nullptr) {
if (raw_ && raw_->dec()) {
raw_->destroy_data();
auto deleter = std::move(static_cast<DeleterT &>(*raw_));
deleter(raw_);
}
raw_ = new_raw;
}
template <class... ArgsT>
static SharedPtr<T, DeleterT> create(ArgsT &&...args) {
auto raw = make_unique<Raw>(DeleterT());
raw->init_data(std::forward<ArgsT>(args)...);
return SharedPtr<T, DeleterT>(raw.release());
}
template <class D, class... ArgsT>
static SharedPtr<T, DeleterT> create_with_deleter(D &&d, ArgsT &&...args) {
auto raw = make_unique<Raw>(std::forward<D>(d));
raw->init_data(std::forward<ArgsT>(args)...);
return SharedPtr<T, DeleterT>(raw.release());
}
bool operator==(const SharedPtr<T, DeleterT> &other) const {
return raw_ == other.raw_;
}
private:
Raw *raw_{nullptr};
};
} // namespace detail
template <class DataT>
class SharedObjectPool {
class Deleter;
public:
using Ptr = detail::SharedPtr<DataT, Deleter>;
SharedObjectPool() = default;
SharedObjectPool(const SharedObjectPool &) = delete;
SharedObjectPool &operator=(const SharedObjectPool &) = delete;
SharedObjectPool(SharedObjectPool &&) = delete;
SharedObjectPool &operator=(SharedObjectPool &&) = delete;
~SharedObjectPool() {
free_queue_.pop_all(free_queue_reader_);
size_t free_cnt = 0;
while (free_queue_reader_.read()) {
free_cnt++;
}
LOG_CHECK(free_cnt == allocated_.size()) << free_cnt << " " << allocated_.size();
}
template <class... ArgsT>
Ptr alloc(ArgsT &&...args) {
auto *raw = alloc_raw();
raw->init_data(std::forward<ArgsT>(args)...);
return Ptr(raw);
}
size_t total_size() const {
return allocated_.size();
}
uint64 calc_free_size() {
free_queue_.pop_all(free_queue_reader_);
return free_queue_reader_.calc_size();
}
// non-thread-safe
template <class F>
void for_each(F &&f) {
for (auto &raw : allocated_) {
if (raw->use_cnt() > 0) {
f(raw->data());
}
}
}
private:
using Raw = typename Ptr::Raw;
Raw *alloc_raw() {
free_queue_.pop_all(free_queue_reader_);
auto *raw = free_queue_reader_.read().get();
if (raw) {
return raw;
}
allocated_.push_back(make_unique<Raw>(deleter()));
return allocated_.back().get();
}
void free_raw(Raw *raw) {
free_queue_.push(Node{raw});
}
class Node {
public:
Node() = default;
explicit Node(Raw *raw) : raw_(raw) {
}
MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
return raw_->to_mpsc_link_queue_node();
}
static Node from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) {
return Node{Raw::from_mpsc_link_queue_node(node)};
}
Raw *get() const {
return raw_;
}
explicit operator bool() const noexcept {
return raw_ != nullptr;
}
private:
Raw *raw_{nullptr};
};
class Deleter {
public:
explicit Deleter(SharedObjectPool<DataT> *pool) : pool_(pool) {
}
void operator()(Raw *raw) {
pool_->free_raw(raw);
};
private:
SharedObjectPool<DataT> *pool_;
};
friend class Deleter;
Deleter deleter() {
return Deleter(this);
}
std::vector<unique_ptr<Raw>> allocated_;
MpscLinkQueue<Node> free_queue_;
typename MpscLinkQueue<Node>::Reader free_queue_reader_;
};
} // namespace td

17
td/tdutils/td/utils/SharedSlice.cpp Normal file
View File

@@ -0,0 +1,17 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/SharedSlice.h"
#include "td/utils/buffer.h"
namespace td {
BufferSlice SharedSlice::clone_as_buffer_slice() const {
return BufferSlice{as_slice()};
}
} // namespace td

382
td/tdutils/td/utils/SharedSlice.h Normal file
View File

@@ -0,0 +1,382 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include <atomic>
#include <memory>
#include <new>
#include <type_traits>
namespace td {
namespace detail {
struct SharedSliceHeader {
explicit SharedSliceHeader(size_t size) : size_{size} {
}
void inc() {
refcnt_.fetch_add(1, std::memory_order_relaxed);
}
bool dec() {
return refcnt_.fetch_sub(1, std::memory_order_acq_rel) == 1;
}
bool is_unique() const {
// NB: race if std::memory_order_relaxed is used
// reader may see a change by a new writer
return refcnt_.load(std::memory_order_acquire) == 1;
}
size_t size() const {
return size_;
}
private:
std::atomic<uint64> refcnt_{1};
size_t size_;
};
struct UniqueSliceHeader {
explicit UniqueSliceHeader(size_t size) : size_{size} {
}
void inc() {
}
bool dec() {
return true;
}
bool is_unique() const {
return true;
}
size_t size() const {
return size_;
}
private:
size_t size_;
};
template <class HeaderT, bool zero_on_destruct = false>
class UnsafeSharedSlice {
public:
UnsafeSharedSlice() = default;
UnsafeSharedSlice clone() const {
if (is_null()) {
return UnsafeSharedSlice();
}
header()->inc();
return UnsafeSharedSlice(ptr_.get());
}
bool is_null() const {
return !ptr_;
}
bool is_unique() const {
if (is_null()) {
return true;
}
return header()->is_unique();
}
MutableSlice as_mutable_slice() {
if (is_null()) {
return MutableSlice();
}
return MutableSlice(ptr_.get() + sizeof(HeaderT), header()->size());
}
Slice as_slice() const {
if (is_null()) {
return Slice();
}
return Slice(ptr_.get() + sizeof(HeaderT), header()->size());
}
size_t size() const {
if (is_null()) {
return 0;
}
return header()->size();
}
static UnsafeSharedSlice create(size_t size) {
static_assert(std::is_standard_layout<HeaderT>::value, "HeaderT must have statdard layout");
auto ptr = std::make_unique<char[]>(sizeof(HeaderT) + size);
auto header_ptr = new (ptr.get()) HeaderT(size);
CHECK(header_ptr == reinterpret_cast<HeaderT *>(ptr.get()));
return UnsafeSharedSlice(std::move(ptr));
}
static UnsafeSharedSlice create(Slice slice) {
auto res = create(slice.size());
res.as_mutable_slice().copy_from(slice);
return res;
}
void clear() {
ptr_.reset();
}
private:
explicit UnsafeSharedSlice(char *ptr) : ptr_(ptr) {
}
explicit UnsafeSharedSlice(std::unique_ptr<char[]> from) : ptr_(from.release()) {
}
HeaderT *header() const {
return reinterpret_cast<HeaderT *>(ptr_.get());
}
struct SharedSliceDestructor {
void operator()(char *ptr) {
auto header = reinterpret_cast<HeaderT *>(ptr);
if (header->dec()) {
if (zero_on_destruct) {
MutableSlice(ptr, sizeof(HeaderT) + header->size()).fill_zero_secure();
}
std::default_delete<char[]>()(ptr);
}
}
};
std::unique_ptr<char[], SharedSliceDestructor> ptr_;
};
} // namespace detail
class BufferSlice;
class UniqueSharedSlice;
class SharedSlice {
using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
public:
SharedSlice() = default;
explicit SharedSlice(Slice slice) : impl_(Impl::create(slice)) {
}
explicit SharedSlice(UniqueSharedSlice from);
SharedSlice(const char *ptr, size_t size) : SharedSlice(Slice(ptr, size)) {
}
SharedSlice clone() const {
return SharedSlice(impl_.clone());
}
Slice as_slice() const {
return impl_.as_slice();
}
BufferSlice clone_as_buffer_slice() const;
operator Slice() const {
return as_slice();
}
// like in std::string
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
friend class UniqueSharedSlice;
explicit SharedSlice(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
class UniqueSharedSlice {
using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
public:
UniqueSharedSlice() = default;
explicit UniqueSharedSlice(size_t size) : impl_(Impl::create(size)) {
}
explicit UniqueSharedSlice(Slice slice) : impl_(Impl::create(slice)) {
}
UniqueSharedSlice(const char *ptr, size_t size) : UniqueSharedSlice(Slice(ptr, size)) {
}
explicit UniqueSharedSlice(SharedSlice from) : impl_() {
if (from.impl_.is_unique()) {
impl_ = std::move(from.impl_);
} else {
impl_ = Impl::create(from.as_slice());
}
}
UniqueSharedSlice copy() const {
return UniqueSharedSlice(as_slice());
}
Slice as_slice() const {
return impl_.as_slice();
}
MutableSlice as_mutable_slice() {
return impl_.as_mutable_slice();
}
operator Slice() const {
return as_slice();
}
// like in std::string
char *data() {
return as_mutable_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
friend class SharedSlice;
explicit UniqueSharedSlice(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
inline SharedSlice::SharedSlice(UniqueSharedSlice from) : impl_(std::move(from.impl_)) {
}
template <bool zero_on_destruct>
class UniqueSliceImpl {
using Impl = detail::UnsafeSharedSlice<detail::UniqueSliceHeader, zero_on_destruct>;
public:
UniqueSliceImpl() = default;
explicit UniqueSliceImpl(size_t size) : impl_(Impl::create(size)) {
}
UniqueSliceImpl(size_t size, char c) : impl_(Impl::create(size)) {
as_mutable_slice().fill(c);
}
explicit UniqueSliceImpl(Slice slice) : impl_(Impl::create(slice)) {
}
UniqueSliceImpl(const char *ptr, size_t size) : UniqueSliceImpl(Slice(ptr, size)) {
}
UniqueSliceImpl copy() const {
return UniqueSliceImpl(as_slice());
}
Slice as_slice() const {
return impl_.as_slice();
}
MutableSlice as_mutable_slice() {
return impl_.as_mutable_slice();
}
operator Slice() const {
return as_slice();
}
// like in std::string
char *data() {
return as_mutable_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
explicit UniqueSliceImpl(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
using UniqueSlice = UniqueSliceImpl<false>;
using SecureString = UniqueSliceImpl<true>;
inline MutableSlice as_mutable_slice(UniqueSharedSlice &unique_shared_slice) {
return unique_shared_slice.as_mutable_slice();
}
inline MutableSlice as_mutable_slice(UniqueSlice &unique_slice) {
return unique_slice.as_mutable_slice();
}
inline MutableSlice as_mutable_slice(SecureString &secure_string) {
return secure_string.as_mutable_slice();
}
} // namespace td

212
td/tdutils/td/utils/Slice-decl.h Normal file
View File

@@ -0,0 +1,212 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <type_traits>
namespace td {
class Slice;
class MutableSlice {
char *s_;
size_t len_;
struct private_tag {};
public:
MutableSlice();
MutableSlice(char *s, size_t len);
MutableSlice(unsigned char *s, size_t len);
MutableSlice(string &s);
template <class T>
explicit MutableSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag> = {});
MutableSlice(char *s, char *t);
MutableSlice(unsigned char *s, unsigned char *t);
template <size_t N>
constexpr MutableSlice(char (&)[N]) = delete;
bool empty() const;
size_t size() const;
MutableSlice &remove_prefix(size_t prefix_len);
MutableSlice &remove_suffix(size_t suffix_len);
MutableSlice &truncate(size_t size);
MutableSlice copy() const;
char *data() const;
char *begin() const;
unsigned char *ubegin() const;
char *end() const;
unsigned char *uend() const;
string str() const;
MutableSlice substr(size_t from) const;
MutableSlice substr(size_t from, size_t size) const;
size_t find(char c) const;
size_t rfind(char c) const;
void fill(char c);
void fill_zero();
void fill_zero_secure();
void copy_from(Slice from);
char &back();
char &operator[](size_t i);
static const size_t npos = static_cast<size_t>(-1);
};
class Slice {
const char *s_;
size_t len_;
struct private_tag {};
public:
Slice();
Slice(const MutableSlice &other);
Slice(const char *s, size_t len);
Slice(const unsigned char *s, size_t len);
Slice(const string &s);
template <class T>
explicit Slice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag> = {});
template <class T>
explicit Slice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag> = {});
Slice(const char *s, const char *t);
Slice(const unsigned char *s, const unsigned char *t);
template <size_t N>
constexpr Slice(char (&)[N]) = delete;
template <size_t N>
constexpr Slice(const char (&a)[N]) : s_(a), len_(N - 1) {
}
Slice &operator=(string &&) = delete;
template <size_t N>
constexpr Slice &operator=(char (&)[N]) = delete;
template <size_t N>
constexpr Slice &operator=(const char (&a)[N]) {
s_ = a;
len_ = N - 1;
return *this;
}
bool empty() const;
size_t size() const;
Slice &remove_prefix(size_t prefix_len);
Slice &remove_suffix(size_t suffix_len);
Slice &truncate(size_t size);
Slice copy() const;
const char *data() const;
const char *begin() const;
const unsigned char *ubegin() const;
const char *end() const;
const unsigned char *uend() const;
string str() const;
Slice substr(size_t from) const;
Slice substr(size_t from, size_t size) const;
size_t find(char c) const;
size_t rfind(char c) const;
char back() const;
char operator[](size_t i) const;
static const size_t npos = static_cast<size_t>(-1);
};
bool operator==(const Slice &a, const Slice &b);
bool operator!=(const Slice &a, const Slice &b);
bool operator<(const Slice &a, const Slice &b);
class MutableCSlice : public MutableSlice {
struct private_tag {};
MutableSlice &remove_suffix(size_t suffix_len) = delete;
MutableSlice &truncate(size_t size) = delete;
public:
MutableCSlice() = delete;
MutableCSlice(string &s) : MutableSlice(s) {
}
template <class T>
explicit MutableCSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag> = {}) : MutableSlice(s) {
}
MutableCSlice(char *s, char *t);
template <size_t N>
constexpr MutableCSlice(char (&)[N]) = delete;
const char *c_str() const {
return begin();
}
};
class CSlice : public Slice {
struct private_tag {};
Slice &remove_suffix(size_t suffix_len) = delete;
Slice &truncate(size_t size) = delete;
public:
explicit CSlice(const MutableSlice &other) : Slice(other) {
}
CSlice(const MutableCSlice &other) : Slice(other.begin(), other.size()) {
}
CSlice(const string &s) : Slice(s) {
}
template <class T>
explicit CSlice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag> = {})
: Slice(s) {
}
template <class T>
explicit CSlice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag> = {})
: Slice(s) {
}
CSlice(const char *s, const char *t);
template <size_t N>
constexpr CSlice(char (&)[N]) = delete;
template <size_t N>
constexpr CSlice(const char (&a)[N]) : Slice(a) {
}
CSlice() : CSlice("") {
}
CSlice &operator=(string &&) = delete;
template <size_t N>
constexpr CSlice &operator=(char (&)[N]) = delete;
template <size_t N>
constexpr CSlice &operator=(const char (&a)[N]) {
this->Slice::operator=(a);
return *this;
}
const char *c_str() const {
return begin();
}
};
struct SliceHash {
uint32 operator()(Slice slice) const;
};
} // namespace td

34
td/tdutils/td/utils/Slice.cpp Normal file
View File

@@ -0,0 +1,34 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/Slice.h"
#if TD_HAVE_OPENSSL
#include <openssl/crypto.h>
#endif
namespace td {
void MutableSlice::fill(char c) {
std::memset(data(), c, size());
}
void MutableSlice::fill_zero() {
fill('\0');
}
void MutableSlice::fill_zero_secure() {
#if TD_HAVE_OPENSSL
OPENSSL_cleanse(begin(), size());
#else
volatile char *ptr = begin();
for (size_t i = 0; i < size(); i++) {
ptr[i] = '\0';
}
#endif
}
} // namespace td

323
td/tdutils/td/utils/Slice.h Normal file
View File

@@ -0,0 +1,323 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice-decl.h"
#include <cstring>
#include <type_traits>
namespace td {
inline MutableSlice::MutableSlice() : s_(const_cast<char *>("")), len_(0) {
}
inline MutableSlice::MutableSlice(char *s, size_t len) : s_(s), len_(len) {
CHECK(s_ != nullptr);
}
inline MutableSlice::MutableSlice(unsigned char *s, size_t len) : s_(reinterpret_cast<char *>(s)), len_(len) {
CHECK(s_ != nullptr);
}
inline MutableSlice::MutableSlice(string &s) : s_(&s[0]), len_(s.size()) {
}
template <class T>
MutableSlice::MutableSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag>) : s_(s) {
CHECK(s_ != nullptr);
len_ = std::strlen(s_);
}
inline MutableSlice::MutableSlice(char *s, char *t) : MutableSlice(s, t - s) {
}
inline MutableSlice::MutableSlice(unsigned char *s, unsigned char *t) : MutableSlice(s, t - s) {
}
inline size_t MutableSlice::size() const {
return len_;
}
inline MutableSlice &MutableSlice::remove_prefix(size_t prefix_len) {
CHECK(prefix_len <= len_);
s_ += prefix_len;
len_ -= prefix_len;
return *this;
}
inline MutableSlice &MutableSlice::remove_suffix(size_t suffix_len) {
CHECK(suffix_len <= len_);
len_ -= suffix_len;
return *this;
}
inline MutableSlice &MutableSlice::truncate(size_t size) {
if (len_ > size) {
len_ = size;
}
return *this;
}
inline MutableSlice MutableSlice::copy() const {
return *this;
}
inline bool MutableSlice::empty() const {
return len_ == 0;
}
inline char *MutableSlice::data() const {
return s_;
}
inline char *MutableSlice::begin() const {
return s_;
}
inline unsigned char *MutableSlice::ubegin() const {
return reinterpret_cast<unsigned char *>(s_);
}
inline char *MutableSlice::end() const {
return s_ + len_;
}
inline unsigned char *MutableSlice::uend() const {
return reinterpret_cast<unsigned char *>(s_) + len_;
}
inline string MutableSlice::str() const {
return string(begin(), size());
}
inline MutableSlice MutableSlice::substr(size_t from) const {
CHECK(from <= len_);
return MutableSlice(s_ + from, len_ - from);
}
inline MutableSlice MutableSlice::substr(size_t from, size_t size) const {
CHECK(from <= len_);
return MutableSlice(s_ + from, min(size, len_ - from));
}
inline size_t MutableSlice::find(char c) const {
for (size_t pos = 0; pos < len_; pos++) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline size_t MutableSlice::rfind(char c) const {
for (size_t pos = len_; pos-- > 0;) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline void MutableSlice::copy_from(Slice from) {
CHECK(size() >= from.size());
std::memcpy(ubegin(), from.ubegin(), from.size());
}
inline char &MutableSlice::back() {
CHECK(1 <= len_);
return s_[len_ - 1];
}
inline char &MutableSlice::operator[](size_t i) {
return s_[i];
}
inline Slice::Slice() : s_(""), len_(0) {
}
inline Slice::Slice(const MutableSlice &other) : s_(other.begin()), len_(other.size()) {
}
inline Slice::Slice(const char *s, size_t len) : s_(s), len_(len) {
CHECK(s_ != nullptr);
}
inline Slice::Slice(const unsigned char *s, size_t len) : s_(reinterpret_cast<const char *>(s)), len_(len) {
CHECK(s_ != nullptr);
}
inline Slice::Slice(const string &s) : s_(s.c_str()), len_(s.size()) {
}
template <class T>
Slice::Slice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag>) : s_(s) {
CHECK(s_ != nullptr);
len_ = std::strlen(s_);
}
template <class T>
Slice::Slice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag>) : s_(s) {
CHECK(s_ != nullptr);
len_ = std::strlen(s_);
}
inline Slice::Slice(const char *s, const char *t) : s_(s), len_(t - s) {
CHECK(s_ != nullptr);
}
inline Slice::Slice(const unsigned char *s, const unsigned char *t)
: s_(reinterpret_cast<const char *>(s)), len_(t - s) {
CHECK(s_ != nullptr);
}
inline size_t Slice::size() const {
return len_;
}
inline Slice &Slice::remove_prefix(size_t prefix_len) {
CHECK(prefix_len <= len_);
s_ += prefix_len;
len_ -= prefix_len;
return *this;
}
inline Slice &Slice::remove_suffix(size_t suffix_len) {
CHECK(suffix_len <= len_);
len_ -= suffix_len;
return *this;
}
inline Slice &Slice::truncate(size_t size) {
if (len_ > size) {
len_ = size;
}
return *this;
}
inline Slice Slice::copy() const {
return *this;
}
inline bool Slice::empty() const {
return len_ == 0;
}
inline const char *Slice::data() const {
return s_;
}
inline const char *Slice::begin() const {
return s_;
}
inline const unsigned char *Slice::ubegin() const {
return reinterpret_cast<const unsigned char *>(s_);
}
inline const char *Slice::end() const {
return s_ + len_;
}
inline const unsigned char *Slice::uend() const {
return reinterpret_cast<const unsigned char *>(s_) + len_;
}
inline string Slice::str() const {
return string(begin(), size());
}
inline Slice Slice::substr(size_t from) const {
CHECK(from <= len_);
return Slice(s_ + from, len_ - from);
}
inline Slice Slice::substr(size_t from, size_t size) const {
CHECK(from <= len_);
return Slice(s_ + from, min(size, len_ - from));
}
inline size_t Slice::find(char c) const {
for (size_t pos = 0; pos < len_; pos++) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline size_t Slice::rfind(char c) const {
for (size_t pos = len_; pos-- > 0;) {
if (s_[pos] == c) {
return pos;
}
}
return npos;
}
inline char Slice::back() const {
CHECK(1 <= len_);
return s_[len_ - 1];
}
inline char Slice::operator[](size_t i) const {
return s_[i];
}
inline bool operator==(const Slice &a, const Slice &b) {
return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size()) == 0;
}
inline bool operator!=(const Slice &a, const Slice &b) {
return !(a == b);
}
inline bool operator<(const Slice &a, const Slice &b) {
auto x = std::memcmp(a.data(), b.data(), td::min(a.size(), b.size()));
if (x == 0) {
return a.size() < b.size();
}
return x < 0;
}
inline MutableCSlice::MutableCSlice(char *s, char *t) : MutableSlice(s, t) {
CHECK(*t == '\0');
}
inline CSlice::CSlice(const char *s, const char *t) : Slice(s, t) {
CHECK(*t == '\0');
}
inline uint32 SliceHash::operator()(Slice slice) const {
// simple string hash
uint32 result = 0;
constexpr uint32 MUL = 123456789;
for (auto c : slice) {
result = result * MUL + c;
}
return result;
}
inline Slice as_slice(Slice slice) {
return slice;
}
inline Slice as_slice(MutableSlice slice) {
return slice;
}
inline Slice as_slice(const string &str) {
return str;
}
inline MutableSlice as_mutable_slice(MutableSlice slice) {
return slice;
}
inline MutableSlice as_mutable_slice(string &str) {
return str;
}
} // namespace td

57
td/tdutils/td/utils/SliceBuilder.h Normal file
View File

@@ -0,0 +1,57 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/StringBuilder.h"
#define PSLICE() ::td::detail::Slicify() & ::td::SliceBuilder().ref()
#define PSTRING() ::td::detail::Stringify() & ::td::SliceBuilder().ref()
namespace td {
class SliceBuilder {
public:
template <class T>
SliceBuilder &operator<<(T &&other) {
sb_ << other;
return *this;
}
MutableCSlice as_cslice() {
return sb_.as_cslice();
}
SliceBuilder &ref() {
return *this;
}
private:
static const size_t DEFAULT_BUFFER_SIZE = 1024;
decltype(StackAllocator::alloc(0)) buffer_ = StackAllocator::alloc(DEFAULT_BUFFER_SIZE);
StringBuilder sb_ = StringBuilder(buffer_.as_slice(), true);
};
namespace detail {
class Slicify {
public:
CSlice operator&(SliceBuilder &slice_builder) {
return slice_builder.as_cslice();
}
};
class Stringify {
public:
string operator&(SliceBuilder &slice_builder) {
return slice_builder.as_cslice().str();
}
};
} // namespace detail
} // namespace td

158
td/tdutils/td/utils/Span.h Normal file
View File

@@ -0,0 +1,158 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include <array>
#include <iterator>
namespace td {
namespace detail {
template <class T, class InnerT>
class SpanImpl {
InnerT *data_{nullptr};
size_t size_{0};
public:
SpanImpl() = default;
SpanImpl(InnerT *data, size_t size) : data_(data), size_(size) {
}
SpanImpl(InnerT &data) : SpanImpl(&data, 1) {
}
template <class OtherInnerT>
SpanImpl(const SpanImpl<T, OtherInnerT> &other) : SpanImpl(other.data(), other.size()) {
}
template <size_t N>
SpanImpl(const std::array<T, N> &arr) : SpanImpl(arr.data(), arr.size()) {
}
template <size_t N>
SpanImpl(std::array<T, N> &arr) : SpanImpl(arr.data(), arr.size()) {
}
template <size_t N>
SpanImpl(const T (&arr)[N]) : SpanImpl(arr, N) {
}
template <size_t N>
SpanImpl(T (&arr)[N]) : SpanImpl(arr, N) {
}
SpanImpl(const vector<T> &v) : SpanImpl(v.data(), v.size()) {
}
SpanImpl(vector<T> &v) : SpanImpl(v.data(), v.size()) {
}
template <class OtherInnerT>
SpanImpl &operator=(const SpanImpl<T, OtherInnerT> &other) {
SpanImpl copy{other};
*this = copy;
}
template <class OtherInnerT>
bool operator==(const SpanImpl<T, OtherInnerT> &other) const {
if (size() != other.size()) {
return false;
}
for (size_t i = 0; i < size(); i++) {
if (!((*this)[i] == other[i])) {
return false;
}
}
return true;
}
InnerT &operator[](size_t i) {
DCHECK(i < size());
return data_[i];
}
const InnerT &operator[](size_t i) const {
DCHECK(i < size());
return data_[i];
}
InnerT &back() {
DCHECK(!empty());
return data_[size() - 1];
}
const InnerT &back() const {
DCHECK(!empty());
return data_[size() - 1];
}
InnerT &front() {
DCHECK(!empty());
return data_[0];
}
const InnerT &front() const {
DCHECK(!empty());
return data_[0];
}
InnerT *data() const {
return data_;
}
InnerT *begin() const {
return data_;
}
InnerT *end() const {
return data_ + size_;
}
std::reverse_iterator<InnerT *> rbegin() const {
return std::reverse_iterator<InnerT *>(end());
}
std::reverse_iterator<InnerT *> rend() const {
return std::reverse_iterator<InnerT *>(begin());
}
size_t size() const {
return size_;
}
bool empty() const {
return size() == 0;
}
SpanImpl &truncate(size_t size) {
if (size < size_) {
size_ = size;
}
return *this;
}
SpanImpl substr(size_t offset) const {
CHECK(offset <= size_);
return SpanImpl(begin() + offset, size_ - offset);
}
SpanImpl substr(size_t offset, size_t size) const {
CHECK(offset <= size_);
CHECK(size_ - offset >= size);
return SpanImpl(begin() + offset, size);
}
};
} // namespace detail
template <class T>
using Span = detail::SpanImpl<T, const T>;
template <class T>
using MutableSpan = detail::SpanImpl<T, T>;
template <class T>
Span<T> as_span(const std::vector<T> &vec) {
return Span<T>(vec);
}
template <class T>
MutableSpan<T> as_mutable_span(std::vector<T> &vec) {
return MutableSpan<T>(vec);
}
} // namespace td

60
td/tdutils/td/utils/SpinLock.h Normal file
View File

@@ -0,0 +1,60 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/port/sleep.h"
#include <atomic>
#include <memory>
namespace td {
class SpinLock {
struct Unlock {
void operator()(SpinLock *ptr) {
ptr->unlock();
}
};
class InfBackoff {
int cnt = 0;
public:
bool next() {
cnt++;
if (cnt < 50) {
//TODO pause
return true;
} else {
usleep_for(1);
return true;
}
}
};
public:
using Lock = std::unique_ptr<SpinLock, Unlock>;
Lock lock() {
InfBackoff backoff;
while (!try_lock()) {
backoff.next();
}
return Lock(this);
}
bool try_lock() {
return !flag_.test_and_set(std::memory_order_acquire);
}
private:
std::atomic_flag flag_ = ATOMIC_FLAG_INIT;
void unlock() {
flag_.clear(std::memory_order_release);
}
};
} // namespace td

80
td/tdutils/td/utils/StackAllocator.cpp Normal file
View File

@@ -0,0 +1,80 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/StackAllocator.h"
#include "td/utils/port/thread_local.h"
#include <array>
#include <cstdlib>
namespace td {
namespace {
class ArrayAllocator final : public StackAllocator::AllocatorImpl {
static const size_t MEM_SIZE = 1024 * 1024;
std::array<char, MEM_SIZE> mem;
size_t pos{0};
MutableSlice allocate(size_t size) final {
if (size > MEM_SIZE) {
std::abort(); // too much memory requested
}
char *res = mem.data() + pos;
pos += (size + 7) & -8;
if (pos > MEM_SIZE) {
std::abort(); // memory is over
}
return {res, size};
}
void free_ptr(char *ptr, size_t size) final {
size = (size + 7) & -8;
if (size > pos || ptr != mem.data() + (pos - size)) {
std::abort(); // shouldn't happen
}
pos -= size;
}
public:
~ArrayAllocator() final {
if (pos != 0) {
std::abort(); // shouldn't happen
}
}
};
class NewAllocator final : public StackAllocator::AllocatorImpl {
MutableSlice allocate(size_t size) final {
return {new char[size], size};
}
void free_ptr(char *ptr, size_t size) final {
delete[] ptr;
}
public:
~NewAllocator() final = default;
};
} // namespace
StackAllocator::Ptr::~Ptr() {
if (!slice_.empty()) {
allocator_->free_ptr(slice_.data(), slice_.size());
}
}
StackAllocator::AllocatorImpl *StackAllocator::impl() {
if (get_thread_id() != 0) {
static TD_THREAD_LOCAL ArrayAllocator *array_allocator; // static zero-initialized
init_thread_local<ArrayAllocator>(array_allocator);
return array_allocator;
} else {
static NewAllocator new_allocator;
return &new_allocator;
}
}
} // namespace td

61
td/tdutils/td/utils/StackAllocator.h Normal file
View File

@@ -0,0 +1,61 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
namespace td {
class StackAllocator {
public:
class AllocatorImpl {
public:
AllocatorImpl() = default;
AllocatorImpl(const AllocatorImpl &) = delete;
AllocatorImpl &operator=(const AllocatorImpl &) = delete;
AllocatorImpl(AllocatorImpl &&) = delete;
AllocatorImpl &operator=(AllocatorImpl &&) = delete;
virtual ~AllocatorImpl() = default;
virtual MutableSlice allocate(size_t size) = 0;
virtual void free_ptr(char *ptr, size_t size) = 0;
};
private:
class Ptr {
public:
Ptr(AllocatorImpl *allocator, size_t size) : allocator_(allocator), slice_(allocator_->allocate(size)) {
}
Ptr(const Ptr &) = delete;
Ptr &operator=(const Ptr &) = delete;
Ptr(Ptr &&other) noexcept : allocator_(other.allocator_), slice_(other.slice_) {
other.allocator_ = nullptr;
other.slice_ = MutableSlice();
}
Ptr &operator=(Ptr &&) = delete;
~Ptr();
MutableSlice as_slice() const {
return slice_;
}
private:
AllocatorImpl *allocator_;
MutableSlice slice_;
};
static AllocatorImpl *impl();
public:
static Ptr alloc(size_t size) {
return Ptr(impl(), size);
}
};
} // namespace td

96
td/tdutils/td/utils/Status.cpp Normal file
View File

@@ -0,0 +1,96 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/Status.h"
#include "td/utils/SliceBuilder.h"
#if TD_PORT_WINDOWS
#include "td/utils/port/wstring_convert.h"
#endif
#if TD_PORT_POSIX
#include "td/utils/port/thread_local.h"
#include <string.h>
#include <cstring>
#endif
namespace td {
#if TD_PORT_POSIX
CSlice strerror_safe(int code) {
const size_t size = 1000;
static TD_THREAD_LOCAL char *buf;
init_thread_local<char[]>(buf, size);
#if !defined(__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE)
strerror_r(code, buf, size);
return CSlice(buf, buf + std::strlen(buf));
#else
return CSlice(strerror_r(code, buf, size));
#endif
}
#endif
#if TD_PORT_WINDOWS
string winerror_to_string(int code) {
const size_t size = 1000;
wchar_t wbuf[size];
auto res_size = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, code, 0, wbuf, size - 1, nullptr);
if (res_size == 0) {
return "Unknown Windows error";
}
while (res_size != 0 && (wbuf[res_size - 1] == '\n' || wbuf[res_size - 1] == '\r')) {
res_size--;
}
auto error_message = from_wstring(wbuf, res_size);
if (error_message.is_error()) {
return "Invalid Windows error";
}
return error_message.move_as_ok();
}
#endif
Status Status::move_as_error_prefix(Slice prefix) const {
CHECK(is_error());
return move_as_error_prefix_unsafe(prefix);
}
Status Status::move_as_error_prefix_unsafe(Slice prefix) const {
Info info = get_info();
switch (info.error_type) {
case ErrorType::General:
return Error(code(), PSLICE() << prefix << message());
case ErrorType::Os:
return Status(false, ErrorType::Os, code(), PSLICE() << prefix << message());
default:
UNREACHABLE();
return {};
}
}
Status Status::move_as_error_suffix(Slice suffix) const {
CHECK(is_error());
return move_as_error_suffix_unsafe(suffix);
}
Status Status::move_as_error_suffix_unsafe(Slice suffix) const {
Info info = get_info();
switch (info.error_type) {
case ErrorType::General:
return Error(code(), PSLICE() << message() << suffix);
case ErrorType::Os:
return Status(false, ErrorType::Os, code(), PSLICE() << message() << suffix);
default:
UNREACHABLE();
return {};
}
}
} // namespace td

607
td/tdutils/td/utils/Status.h Normal file
View File

@@ -0,0 +1,607 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/ScopeGuard.h"
#include "td/utils/Slice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/StringBuilder.h"
#include <cerrno>
#include <cstring>
#include <memory>
#include <new>
#include <type_traits>
#include <utility>
#define TRY_STATUS(status) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return try_status.move_as_error_unsafe(); \
} \
}
#define TRY_STATUS_PREFIX(status, prefix) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return try_status.move_as_error_prefix_unsafe(prefix); \
} \
}
#define TRY_STATUS_PROMISE(promise_name, status) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return promise_name.set_error(try_status.move_as_error_unsafe()); \
} \
}
#define TRY_STATUS_PROMISE_PREFIX(promise_name, status, prefix) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return promise_name.set_error(try_status.move_as_error_prefix_unsafe(prefix)); \
} \
}
#define TRY_RESULT(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result)
#define TRY_RESULT_PROMISE(promise_name, name, result) \
TRY_RESULT_PROMISE_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result)
#define TRY_RESULT_ASSIGN(name, result) TRY_RESULT_IMPL(TD_CONCAT(r_response, __LINE__), name, result)
#define TRY_RESULT_PROMISE_ASSIGN(promise_name, name, result) \
TRY_RESULT_PROMISE_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result)
#define TRY_RESULT_PREFIX(name, result, prefix) \
TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix)
#define TRY_RESULT_PREFIX_ASSIGN(name, result, prefix) \
TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
#define TRY_RESULT_PROMISE_PREFIX(promise_name, name, result, prefix) \
TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix)
#define TRY_RESULT_PROMISE_PREFIX_ASSIGN(promise_name, name, result, prefix) \
TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
#define TRY_RESULT_IMPL(r_name, name, result) \
auto r_name = (result); \
if (r_name.is_error()) { \
return r_name.move_as_error_unsafe(); \
} \
name = r_name.move_as_ok_unsafe();
#define TRY_RESULT_PREFIX_IMPL(r_name, name, result, prefix) \
auto r_name = (result); \
if (r_name.is_error()) { \
return r_name.move_as_error_prefix_unsafe(prefix); \
} \
name = r_name.move_as_ok_unsafe();
#define TRY_RESULT_PROMISE_IMPL(promise_name, r_name, name, result) \
auto r_name = (result); \
if (r_name.is_error()) { \
return promise_name.set_error(r_name.move_as_error_unsafe()); \
} \
name = r_name.move_as_ok_unsafe();
#define TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, r_name, name, result, prefix) \
auto r_name = (result); \
if (r_name.is_error()) { \
return promise_name.set_error(r_name.move_as_error_prefix_unsafe(prefix)); \
} \
name = r_name.move_as_ok_unsafe();
#define LOG_STATUS(status) \
{ \
auto log_status = (status); \
if (log_status.is_error()) { \
LOG(ERROR) << log_status.move_as_error_unsafe(); \
} \
}
#ifndef TD_STATUS_NO_ENSURE
#define ensure() ensure_impl(__FILE__, __LINE__)
#define ensure_error() ensure_error_impl(__FILE__, __LINE__)
#endif
#if TD_PORT_POSIX
#define OS_ERROR(message) \
[&] { \
auto saved_errno = errno; \
return ::td::Status::PosixError(saved_errno, (message)); \
}()
#define OS_SOCKET_ERROR(message) OS_ERROR(message)
#elif TD_PORT_WINDOWS
#define OS_ERROR(message) \
[&] { \
auto saved_error = ::GetLastError(); \
return ::td::Status::WindowsError(saved_error, (message)); \
}()
#define OS_SOCKET_ERROR(message) \
[&] { \
auto saved_error = ::WSAGetLastError(); \
return ::td::Status::WindowsError(saved_error, (message)); \
}()
#endif
namespace td {
#if TD_PORT_POSIX
CSlice strerror_safe(int code);
#endif
#if TD_PORT_WINDOWS
string winerror_to_string(int code);
#endif
class Status {
enum class ErrorType : int8 { General, Os };
public:
Status() = default;
bool is_static() const {
if (is_ok()) {
return true;
}
return get_info().static_flag;
}
Status clone() const TD_WARN_UNUSED_RESULT {
if (is_ok()) {
return Status();
}
auto info = get_info();
if (info.static_flag) {
return clone_static(-999);
}
return Status(false, info.error_type, info.error_code, message());
}
static Status OK() TD_WARN_UNUSED_RESULT {
return Status();
}
static Status Error(int err, Slice message = Slice()) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::General, err, message);
}
static Status Error(Slice message) TD_WARN_UNUSED_RESULT {
return Error(0, message);
}
#if TD_PORT_WINDOWS
static Status WindowsError(int saved_error, Slice message) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::Os, saved_error, message);
}
#endif
#if TD_PORT_POSIX
static Status PosixError(int32 saved_errno, Slice message) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::Os, saved_errno, message);
}
#endif
template <int Code>
static Status Error() {
static Status status(true, ErrorType::General, Code, "");
return status.clone_static(Code);
}
StringBuilder &print(StringBuilder &sb) const {
if (is_ok()) {
return sb << "OK";
}
Info info = get_info();
switch (info.error_type) {
case ErrorType::General:
sb << "[Error";
break;
case ErrorType::Os:
#if TD_PORT_POSIX
sb << "[PosixError : " << strerror_safe(info.error_code);
#elif TD_PORT_WINDOWS
sb << "[WindowsError : " << winerror_to_string(info.error_code);
#endif
break;
default:
UNREACHABLE();
break;
}
sb << " : " << code() << " : " << message() << "]";
return sb;
}
string to_string() const {
auto buf = StackAllocator::alloc(4096);
StringBuilder sb(buf.as_slice());
print(sb);
return sb.as_cslice().str();
}
// Default interface
bool is_ok() const TD_WARN_UNUSED_RESULT {
return !is_error();
}
bool is_error() const TD_WARN_UNUSED_RESULT {
return ptr_ != nullptr;
}
#ifdef TD_STATUS_NO_ENSURE
void ensure() const {
if (!is_ok()) {
LOG(FATAL) << "Unexpected Status " << to_string();
}
}
void ensure_error() const {
if (is_ok()) {
LOG(FATAL) << "Unexpected Status::OK";
}
}
#else
void ensure_impl(CSlice file_name, int line) const {
if (!is_ok()) {
LOG(FATAL) << "Unexpected Status " << to_string() << " in file " << file_name << " at line " << line;
}
}
void ensure_error_impl(CSlice file_name, int line) const {
if (is_ok()) {
LOG(FATAL) << "Unexpected Status::OK in file " << file_name << " at line " << line;
}
}
#endif
void ignore() const {
// nop
}
int32 code() const {
if (is_ok()) {
return 0;
}
return get_info().error_code;
}
CSlice message() const {
if (is_ok()) {
return CSlice("OK");
}
return CSlice(ptr_.get() + sizeof(Info));
}
string public_message() const {
if (is_ok()) {
return "OK";
}
Info info = get_info();
switch (info.error_type) {
case ErrorType::General:
return message().str();
case ErrorType::Os:
#if TD_PORT_POSIX
return strerror_safe(info.error_code).str();
#elif TD_PORT_WINDOWS
return winerror_to_string(info.error_code);
#endif
default:
UNREACHABLE();
return "";
}
}
const Status &error() const {
return *this;
}
Status move() TD_WARN_UNUSED_RESULT {
return std::move(*this);
}
Status move_as_error() TD_WARN_UNUSED_RESULT {
return std::move(*this);
}
Status move_as_error_unsafe() TD_WARN_UNUSED_RESULT {
return std::move(*this);
}
Status move_as_ok() = delete;
Status move_as_ok_unsafe() = delete;
Status move_as_error_prefix(const Status &status) const TD_WARN_UNUSED_RESULT {
return status.move_as_error_suffix(message());
}
Status move_as_error_prefix(Slice prefix) const TD_WARN_UNUSED_RESULT;
Status move_as_error_prefix_unsafe(Slice prefix) const TD_WARN_UNUSED_RESULT;
Status move_as_error_suffix(Slice suffix) const TD_WARN_UNUSED_RESULT;
Status move_as_error_suffix_unsafe(Slice suffix) const TD_WARN_UNUSED_RESULT;
private:
struct Info {
bool static_flag : 1;
signed int error_code : 23;
ErrorType error_type;
};
struct Deleter {
void operator()(char *ptr) {
if (!get_info(ptr).static_flag) {
delete[] ptr;
}
}
};
std::unique_ptr<char[], Deleter> ptr_;
Status(Info info, Slice message) {
size_t size = sizeof(Info) + message.size() + 1;
ptr_ = std::unique_ptr<char[], Deleter>(new char[size]);
char *ptr = ptr_.get();
reinterpret_cast<Info *>(ptr)[0] = info;
ptr += sizeof(Info);
std::memcpy(ptr, message.begin(), message.size());
ptr += message.size();
*ptr = 0;
}
Status(bool static_flag, ErrorType error_type, int error_code, Slice message)
: Status(to_info(static_flag, error_type, error_code), message) {
if (static_flag) {
TD_LSAN_IGNORE(ptr_.get());
}
}
Status clone_static(int code) const TD_WARN_UNUSED_RESULT {
LOG_CHECK(ptr_ != nullptr && get_info().static_flag) << ptr_.get() << ' ' << code;
Status result;
result.ptr_ = std::unique_ptr<char[], Deleter>(ptr_.get());
return result;
}
static Info to_info(bool static_flag, ErrorType error_type, int error_code) {
const int MIN_ERROR_CODE = -(1 << 22) + 1;
const int MAX_ERROR_CODE = (1 << 22) - 1;
Info tmp;
tmp.static_flag = static_flag;
tmp.error_type = error_type;
if (error_code < MIN_ERROR_CODE) {
LOG(ERROR) << "Error code value is altered from " << error_code;
error_code = MIN_ERROR_CODE;
}
if (error_code > MAX_ERROR_CODE) {
LOG(ERROR) << "Error code value is altered from " << error_code;
error_code = MAX_ERROR_CODE;
}
#if TD_GCC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
tmp.error_code = error_code;
#if TD_GCC
#pragma GCC diagnostic pop
#endif
CHECK(error_code == tmp.error_code);
return tmp;
}
Info get_info() const {
return get_info(ptr_.get());
}
static Info get_info(char *ptr) {
return reinterpret_cast<Info *>(ptr)[0];
}
};
template <class T = Unit>
class Result {
public:
using ValueT = T;
Result() : status_(Status::Error<-1>()) {
}
template <class S, std::enable_if_t<!std::is_same<std::decay_t<S>, Result>::value, int> = 0>
Result(S &&x) : status_(), value_(std::forward<S>(x)) {
}
struct emplace_t {};
template <class... ArgsT>
Result(emplace_t, ArgsT &&...args) : status_(), value_(std::forward<ArgsT>(args)...) {
}
Result(Status &&status) : status_(std::move(status)) {
CHECK(status_.is_error());
}
Result(const Result &) = delete;
Result &operator=(const Result &) = delete;
Result(Result &&other) noexcept : status_(std::move(other.status_)) {
if (status_.is_ok()) {
new (&value_) T(std::move(other.value_));
other.value_.~T();
}
other.status_ = Status::Error<-2>();
}
Result &operator=(Result &&other) noexcept {
CHECK(this != &other);
if (status_.is_ok()) {
value_.~T();
}
if (other.status_.is_ok()) {
#if TD_GCC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
new (&value_) T(std::move(other.value_));
#if TD_GCC
#pragma GCC diagnostic pop
#endif
other.value_.~T();
}
status_ = std::move(other.status_);
other.status_ = Status::Error<-3>();
return *this;
}
template <class... ArgsT>
void emplace(ArgsT &&...args) {
if (status_.is_ok()) {
value_.~T();
}
new (&value_) T(std::forward<ArgsT>(args)...);
status_ = Status::OK();
}
~Result() {
if (status_.is_ok()) {
value_.~T();
}
}
#ifdef TD_STATUS_NO_ENSURE
void ensure() const {
status_.ensure();
}
void ensure_error() const {
status_.ensure_error();
}
#else
void ensure_impl(CSlice file_name, int line) const {
status_.ensure_impl(file_name, line);
}
void ensure_error_impl(CSlice file_name, int line) const {
status_.ensure_error_impl(file_name, line);
}
#endif
void ignore() const {
status_.ignore();
}
bool is_ok() const {
return status_.is_ok();
}
bool is_error() const {
return status_.is_error();
}
const Status &error() const {
CHECK(status_.is_error());
return status_;
}
Status move_as_error() TD_WARN_UNUSED_RESULT {
CHECK(status_.is_error());
SCOPE_EXIT {
status_ = Status::Error<-4>();
};
return std::move(status_);
}
Status move_as_error_unsafe() TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-5>();
};
return std::move(status_);
}
Status move_as_error_prefix(Slice prefix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-6>();
};
return status_.move_as_error_prefix(prefix);
}
Status move_as_error_prefix_unsafe(Slice prefix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-7>();
};
return status_.move_as_error_prefix_unsafe(prefix);
}
Status move_as_error_prefix(const Status &prefix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-8>();
};
return status_.move_as_error_prefix(prefix);
}
Status move_as_error_suffix(Slice suffix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-9>();
};
return status_.move_as_error_suffix(suffix);
}
Status move_as_error_suffix_unsafe(Slice suffix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-10>();
};
return status_.move_as_error_suffix_unsafe(suffix);
}
const T &ok() const {
LOG_CHECK(status_.is_ok()) << status_;
return value_;
}
T &ok_ref() {
LOG_CHECK(status_.is_ok()) << status_;
return value_;
}
const T &ok_ref() const {
LOG_CHECK(status_.is_ok()) << status_;
return value_;
}
T move_as_ok() {
LOG_CHECK(status_.is_ok()) << status_;
return std::move(value_);
}
T move_as_ok_unsafe() {
return std::move(value_);
}
Result<T> clone() const TD_WARN_UNUSED_RESULT {
if (is_ok()) {
return Result<T>(ok()); // TODO: return clone(ok());
}
return error().clone();
}
void clear() {
*this = Result<T>();
}
template <class F>
Result<decltype(std::declval<F>()(std::declval<T>()))> move_map(F &&f) {
if (is_error()) {
return move_as_error_unsafe();
}
return f(move_as_ok_unsafe());
}
template <class F>
decltype(std::declval<F>()(std::declval<T>())) move_fmap(F &&f) {
if (is_error()) {
return move_as_error_unsafe();
}
return f(move_as_ok_unsafe());
}
private:
Status status_;
union {
T value_;
};
};
template <>
inline Result<Unit>::Result(Status &&status) : status_(std::move(status)) {
// no assert
}
inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) {
return status.print(string_builder);
}
} // namespace td

125
td/tdutils/td/utils/StealingQueue.h Normal file
View File

@@ -0,0 +1,125 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/misc.h"
#include <array>
#include <atomic>
namespace td {
template <class T, size_t N = 256>
class StealingQueue {
public:
static_assert(N > 0 && (N & (N - 1)) == 0, "");
// tries to put a value
// returns if succeeded
// only owner is allowed to to do this
template <class F>
void local_push(T value, F &&overflow_f) {
while (true) {
auto tail = tail_.load(std::memory_order_relaxed);
auto head = head_.load(); // TODO: memory order
if (static_cast<size_t>(tail - head) < N) {
buf_[tail & MASK].store(value, std::memory_order_relaxed);
tail_.store(tail + 1, std::memory_order_release);
return;
}
// queue is full
// TODO: batch insert into global queue?
auto n = N / 2 + 1;
auto new_head = head + n;
if (!head_.compare_exchange_strong(head, new_head)) {
continue;
}
for (size_t i = 0; i < n; i++) {
overflow_f(buf_[(i + head) & MASK].load(std::memory_order_relaxed));
}
overflow_f(value);
return;
}
}
// tries to pop a value
// returns if succeeded
// only owner is allowed to do this
bool local_pop(T &value) {
auto tail = tail_.load(std::memory_order_relaxed);
auto head = head_.load();
if (head == tail) {
return false;
}
value = buf_[head & MASK].load(std::memory_order_relaxed);
return head_.compare_exchange_strong(head, head + 1);
}
bool steal(T &value, StealingQueue<T, N> &other) {
while (true) {
auto tail = tail_.load(std::memory_order_relaxed);
auto head = head_.load(); // TODO: memory order
auto other_head = other.head_.load();
auto other_tail = other.tail_.load(std::memory_order_acquire);
if (other_tail < other_head) {
continue;
}
auto n = narrow_cast<size_t>(other_tail - other_head);
if (n > N) {
continue;
}
n -= n / 2;
n = td::min(n, static_cast<size_t>(head + N - tail));
if (n == 0) {
return false;
}
for (size_t i = 0; i < n; i++) {
buf_[(i + tail) & MASK].store(other.buf_[(i + other_head) & MASK].load(std::memory_order_relaxed),
std::memory_order_relaxed);
}
if (!other.head_.compare_exchange_strong(other_head, other_head + n)) {
continue;
}
n--;
value = buf_[(tail + n) & MASK].load(std::memory_order_relaxed);
tail_.store(tail + n, std::memory_order_release);
return true;
}
}
StealingQueue() {
for (auto &x : buf_) {
// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658
#if TD_GCC && GCC_VERSION <= 40902
x = T();
#else
std::atomic_init(&x, T());
#endif
}
std::atomic_thread_fence(std::memory_order_seq_cst);
}
private:
std::atomic<int64> head_{0};
std::atomic<int64> tail_{0};
static constexpr size_t MASK{N - 1};
std::array<std::atomic<T>, N> buf_;
};
} // namespace td

88
td/tdutils/td/utils/Storer.h Normal file
View File

@@ -0,0 +1,88 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/StorerBase.h"
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/tl_storers.h"
#include <cstring>
#include <limits>
namespace td {
class SliceStorer final : public Storer {
Slice slice;
public:
explicit SliceStorer(Slice slice) : slice(slice) {
}
size_t size() const final {
return slice.size();
}
size_t store(uint8 *ptr) const final {
std::memcpy(ptr, slice.ubegin(), slice.size());
return slice.size();
}
};
inline SliceStorer create_storer(Slice slice) {
return SliceStorer(slice);
}
class ConcatStorer final : public Storer {
const Storer &a_;
const Storer &b_;
public:
ConcatStorer(const Storer &a, const Storer &b) : a_(a), b_(b) {
}
size_t size() const final {
return a_.size() + b_.size();
}
size_t store(uint8 *ptr) const final {
uint8 *ptr_save = ptr;
ptr += a_.store(ptr);
ptr += b_.store(ptr);
return ptr - ptr_save;
}
};
inline ConcatStorer create_storer(const Storer &a, const Storer &b) {
return ConcatStorer(a, b);
}
template <class T>
class DefaultStorer final : public Storer {
public:
explicit DefaultStorer(const T &object) : object_(object) {
}
size_t size() const final {
if (size_ == std::numeric_limits<size_t>::max()) {
size_ = tl_calc_length(object_);
}
return size_;
}
size_t store(uint8 *ptr) const final {
return tl_store_unsafe(object_, ptr);
}
private:
mutable size_t size_ = std::numeric_limits<size_t>::max();
const T &object_;
};
template <class T>
DefaultStorer<T> create_default_storer(const T &from) {
return DefaultStorer<T>(from);
}
} // namespace td

25
td/tdutils/td/utils/StorerBase.h Normal file
View File

@@ -0,0 +1,25 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/int_types.h"
namespace td {
class Storer {
public:
Storer() = default;
Storer(const Storer &) = delete;
Storer &operator=(const Storer &) = delete;
Storer(Storer &&) = default;
Storer &operator=(Storer &&) = default;
virtual ~Storer() = default;
virtual size_t size() const = 0;
virtual size_t store(uint8 *ptr) const TD_WARN_UNUSED_RESULT = 0;
};
} // namespace td

Some files were not shown because too many files have changed in this diff Show More